{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting ./mnist_data/train-images-idx3-ubyte.gz\n",
      "Extracting ./mnist_data/train-labels-idx1-ubyte.gz\n",
      "Extracting ./mnist_data/t10k-images-idx3-ubyte.gz\n",
      "Extracting ./mnist_data/t10k-labels-idx1-ubyte.gz\n",
      "step 100, entropy loss: 2.302212, l2_loss: 184.632568, total loss: 2.315136\n",
      "0.12\n",
      "step 200, entropy loss: 2.296863, l2_loss: 184.613907, total loss: 2.309786\n",
      "0.18\n",
      "step 300, entropy loss: 2.298353, l2_loss: 184.598480, total loss: 2.311275\n",
      "0.14\n",
      "step 400, entropy loss: 2.294134, l2_loss: 184.588104, total loss: 2.307055\n",
      "0.22\n",
      "step 500, entropy loss: 2.295110, l2_loss: 184.581268, total loss: 2.308031\n",
      "0.16\n",
      "step 600, entropy loss: 2.288191, l2_loss: 184.581848, total loss: 2.301112\n",
      "0.24\n",
      "step 700, entropy loss: 2.291620, l2_loss: 184.589188, total loss: 2.304541\n",
      "0.18\n",
      "step 800, entropy loss: 2.294708, l2_loss: 184.603363, total loss: 2.307630\n",
      "0.21\n",
      "step 900, entropy loss: 2.287724, l2_loss: 184.627472, total loss: 2.300648\n",
      "0.19\n",
      "step 1000, entropy loss: 2.291512, l2_loss: 184.660980, total loss: 2.304438\n",
      "0.15\n",
      "0.1905\n",
      "step 1100, entropy loss: 2.272998, l2_loss: 184.695450, total loss: 2.285926\n",
      "0.2\n",
      "step 1200, entropy loss: 2.266403, l2_loss: 184.744492, total loss: 2.279335\n",
      "0.15\n",
      "step 1300, entropy loss: 2.269271, l2_loss: 184.807281, total loss: 2.282208\n",
      "0.23\n",
      "step 1400, entropy loss: 2.260622, l2_loss: 184.872925, total loss: 2.273563\n",
      "0.16\n",
      "step 1500, entropy loss: 2.233076, l2_loss: 184.940582, total loss: 2.246022\n",
      "0.27\n",
      "step 1600, entropy loss: 2.235501, l2_loss: 185.024490, total loss: 2.248453\n",
      "0.22\n",
      "step 1700, entropy loss: 2.217519, l2_loss: 185.123001, total loss: 2.230478\n",
      "0.29\n",
      "step 1800, entropy loss: 2.220648, l2_loss: 185.235443, total loss: 2.233614\n",
      "0.24\n",
      "step 1900, entropy loss: 2.214323, l2_loss: 185.358459, total loss: 2.227298\n",
      "0.23\n",
      "step 2000, entropy loss: 2.183343, l2_loss: 185.498444, total loss: 2.196328\n",
      "0.31\n",
      "0.2191\n",
      "step 2100, entropy loss: 2.192422, l2_loss: 185.654770, total loss: 2.205418\n",
      "0.25\n",
      "step 2200, entropy loss: 2.180218, l2_loss: 185.816772, total loss: 2.193225\n",
      "0.26\n",
      "step 2300, entropy loss: 2.205956, l2_loss: 186.003265, total loss: 2.218976\n",
      "0.11\n",
      "step 2400, entropy loss: 2.219059, l2_loss: 186.213593, total loss: 2.232095\n",
      "0.18\n",
      "step 2500, entropy loss: 2.202178, l2_loss: 186.460098, total loss: 2.215230\n",
      "0.17\n",
      "step 2600, entropy loss: 2.163327, l2_loss: 186.714203, total loss: 2.176397\n",
      "0.21\n",
      "step 2700, entropy loss: 2.129446, l2_loss: 187.002930, total loss: 2.142536\n",
      "0.26\n",
      "step 2800, entropy loss: 2.055472, l2_loss: 187.313278, total loss: 2.068584\n",
      "0.29\n",
      "step 2900, entropy loss: 2.134704, l2_loss: 187.667542, total loss: 2.147840\n",
      "0.17\n",
      "step 3000, entropy loss: 2.056895, l2_loss: 188.051544, total loss: 2.070058\n",
      "0.31\n",
      "0.2612\n",
      "step 3100, entropy loss: 1.986292, l2_loss: 188.472748, total loss: 1.999485\n",
      "0.36\n",
      "step 3200, entropy loss: 1.968134, l2_loss: 188.923782, total loss: 1.981359\n",
      "0.31\n",
      "step 3300, entropy loss: 1.911030, l2_loss: 189.425613, total loss: 1.924290\n",
      "0.3\n",
      "step 3400, entropy loss: 1.948366, l2_loss: 189.959473, total loss: 1.961664\n",
      "0.3\n",
      "step 3500, entropy loss: 1.950653, l2_loss: 190.539917, total loss: 1.963990\n",
      "0.33\n",
      "step 3600, entropy loss: 1.901071, l2_loss: 191.148163, total loss: 1.914452\n",
      "0.37\n",
      "step 3700, entropy loss: 1.817842, l2_loss: 191.818726, total loss: 1.831269\n",
      "0.42\n",
      "step 3800, entropy loss: 1.817488, l2_loss: 192.489502, total loss: 1.830963\n",
      "0.34\n",
      "step 3900, entropy loss: 1.734767, l2_loss: 193.163818, total loss: 1.748288\n",
      "0.35\n",
      "step 4000, entropy loss: 1.716905, l2_loss: 193.897644, total loss: 1.730477\n",
      "0.39\n",
      "0.3514\n",
      "step 4100, entropy loss: 1.838047, l2_loss: 194.619202, total loss: 1.851670\n",
      "0.33\n",
      "step 4200, entropy loss: 1.741028, l2_loss: 195.334885, total loss: 1.754702\n",
      "0.36\n",
      "step 4300, entropy loss: 1.686345, l2_loss: 196.046906, total loss: 1.700068\n",
      "0.34\n",
      "step 4400, entropy loss: 1.675883, l2_loss: 196.727432, total loss: 1.689654\n",
      "0.44\n",
      "step 4500, entropy loss: 1.578225, l2_loss: 197.379044, total loss: 1.592042\n",
      "0.37\n",
      "step 4600, entropy loss: 1.491731, l2_loss: 197.992615, total loss: 1.505591\n",
      "0.37\n",
      "step 4700, entropy loss: 1.682560, l2_loss: 198.604874, total loss: 1.696463\n",
      "0.34\n",
      "step 4800, entropy loss: 1.468626, l2_loss: 199.219330, total loss: 1.482571\n",
      "0.4\n",
      "step 4900, entropy loss: 1.482868, l2_loss: 199.804718, total loss: 1.496854\n",
      "0.43\n",
      "step 5000, entropy loss: 1.517585, l2_loss: 200.417358, total loss: 1.531615\n",
      "0.47\n",
      "0.4757\n",
      "step 5100, entropy loss: 1.558478, l2_loss: 201.138123, total loss: 1.572557\n",
      "0.45\n",
      "step 5200, entropy loss: 1.384681, l2_loss: 201.876312, total loss: 1.398812\n",
      "0.58\n",
      "step 5300, entropy loss: 1.177631, l2_loss: 202.618942, total loss: 1.191814\n",
      "0.54\n",
      "step 5400, entropy loss: 1.327787, l2_loss: 203.420670, total loss: 1.342026\n",
      "0.5\n",
      "step 5500, entropy loss: 1.254209, l2_loss: 204.266571, total loss: 1.268508\n",
      "0.57\n",
      "step 5600, entropy loss: 1.320537, l2_loss: 205.170013, total loss: 1.334899\n",
      "0.59\n",
      "step 5700, entropy loss: 1.209921, l2_loss: 206.182587, total loss: 1.224354\n",
      "0.64\n",
      "step 5800, entropy loss: 1.167200, l2_loss: 207.249176, total loss: 1.181707\n",
      "0.62\n",
      "step 5900, entropy loss: 1.041480, l2_loss: 208.312164, total loss: 1.056062\n",
      "0.65\n",
      "step 6000, entropy loss: 1.159407, l2_loss: 209.431458, total loss: 1.174068\n",
      "0.68\n",
      "0.652\n",
      "step 6100, entropy loss: 0.837819, l2_loss: 210.531830, total loss: 0.852556\n",
      "0.79\n",
      "step 6200, entropy loss: 0.984847, l2_loss: 211.640274, total loss: 0.999662\n",
      "0.69\n",
      "step 6300, entropy loss: 1.049319, l2_loss: 212.709534, total loss: 1.064209\n",
      "0.65\n",
      "step 6400, entropy loss: 0.909063, l2_loss: 213.797882, total loss: 0.924029\n",
      "0.75\n",
      "step 6500, entropy loss: 0.785143, l2_loss: 214.840027, total loss: 0.800182\n",
      "0.82\n",
      "step 6600, entropy loss: 0.938545, l2_loss: 215.877045, total loss: 0.953656\n",
      "0.69\n",
      "step 6700, entropy loss: 0.871987, l2_loss: 216.901123, total loss: 0.887170\n",
      "0.74\n",
      "step 6800, entropy loss: 0.702634, l2_loss: 217.856171, total loss: 0.717884\n",
      "0.78\n",
      "step 6900, entropy loss: 0.795941, l2_loss: 218.829773, total loss: 0.811259\n",
      "0.77\n",
      "step 7000, entropy loss: 0.736456, l2_loss: 219.848877, total loss: 0.751845\n",
      "0.77\n",
      "0.7757\n",
      "step 7100, entropy loss: 0.769390, l2_loss: 220.744614, total loss: 0.784843\n",
      "0.77\n",
      "step 7200, entropy loss: 0.659769, l2_loss: 221.599548, total loss: 0.675281\n",
      "0.79\n",
      "step 7300, entropy loss: 0.672464, l2_loss: 222.496613, total loss: 0.688039\n",
      "0.82\n",
      "step 7400, entropy loss: 0.549626, l2_loss: 223.349716, total loss: 0.565260\n",
      "0.85\n",
      "step 7500, entropy loss: 0.646707, l2_loss: 224.188049, total loss: 0.662400\n",
      "0.84\n",
      "step 7600, entropy loss: 0.643406, l2_loss: 224.990051, total loss: 0.659156\n",
      "0.78\n",
      "step 7700, entropy loss: 0.650414, l2_loss: 225.789948, total loss: 0.666219\n",
      "0.79\n",
      "step 7800, entropy loss: 0.697196, l2_loss: 226.602432, total loss: 0.713059\n",
      "0.79\n",
      "step 7900, entropy loss: 0.594019, l2_loss: 227.321106, total loss: 0.609932\n",
      "0.84\n",
      "step 8000, entropy loss: 0.654627, l2_loss: 228.063019, total loss: 0.670591\n",
      "0.81\n",
      "0.829\n",
      "step 8100, entropy loss: 0.621452, l2_loss: 228.816513, total loss: 0.637469\n",
      "0.82\n",
      "step 8200, entropy loss: 0.506174, l2_loss: 229.562943, total loss: 0.522243\n",
      "0.87\n",
      "step 8300, entropy loss: 0.493853, l2_loss: 230.268143, total loss: 0.509972\n",
      "0.89\n",
      "step 8400, entropy loss: 0.392707, l2_loss: 230.989746, total loss: 0.408877\n",
      "0.91\n",
      "step 8500, entropy loss: 0.566622, l2_loss: 231.705826, total loss: 0.582841\n",
      "0.84\n",
      "step 8600, entropy loss: 0.429784, l2_loss: 232.334534, total loss: 0.446048\n",
      "0.89\n",
      "step 8700, entropy loss: 0.676457, l2_loss: 232.974167, total loss: 0.692765\n",
      "0.85\n",
      "step 8800, entropy loss: 0.583183, l2_loss: 233.634857, total loss: 0.599538\n",
      "0.88\n",
      "step 8900, entropy loss: 0.410942, l2_loss: 234.232086, total loss: 0.427339\n",
      "0.88\n",
      "step 9000, entropy loss: 0.390920, l2_loss: 234.870667, total loss: 0.407361\n",
      "0.88\n",
      "0.8753\n",
      "step 9100, entropy loss: 0.373450, l2_loss: 235.475616, total loss: 0.389933\n",
      "0.9\n",
      "step 9200, entropy loss: 0.372722, l2_loss: 236.066681, total loss: 0.389246\n",
      "0.92\n",
      "step 9300, entropy loss: 0.488046, l2_loss: 236.669342, total loss: 0.504613\n",
      "0.86\n",
      "step 9400, entropy loss: 0.450971, l2_loss: 237.249908, total loss: 0.467578\n",
      "0.89\n",
      "step 9500, entropy loss: 0.475858, l2_loss: 237.794495, total loss: 0.492503\n",
      "0.87\n",
      "step 9600, entropy loss: 0.509508, l2_loss: 238.320831, total loss: 0.526190\n",
      "0.84\n",
      "step 9700, entropy loss: 0.405639, l2_loss: 238.860535, total loss: 0.422359\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.87\n",
      "step 9800, entropy loss: 0.404766, l2_loss: 239.375977, total loss: 0.421523\n",
      "0.87\n",
      "step 9900, entropy loss: 0.378015, l2_loss: 239.894623, total loss: 0.394807\n",
      "0.87\n",
      "step 10000, entropy loss: 0.587733, l2_loss: 240.373108, total loss: 0.604560\n",
      "0.84\n",
      "0.8797\n"
     ]
    }
   ],
   "source": [
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "# Import data\n",
    "data_dir=\"./mnist_data/\"\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "# Define loss and optimizer\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 1.将Conv1的size从(5,5)-->(3,3);Conv2的size从(5,5)-->(3,3)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.304328, l2_loss: 277.378296, total loss: 2.323745\n",
      "0.1\n",
      "step 200, entropy loss: 2.296207, l2_loss: 277.345245, total loss: 2.315621\n",
      "0.21\n",
      "step 300, entropy loss: 2.297585, l2_loss: 277.317932, total loss: 2.316998\n",
      "0.17\n",
      "step 400, entropy loss: 2.289569, l2_loss: 277.292297, total loss: 2.308980\n",
      "0.25\n",
      "step 500, entropy loss: 2.297775, l2_loss: 277.269043, total loss: 2.317184\n",
      "0.19\n",
      "step 600, entropy loss: 2.297868, l2_loss: 277.250061, total loss: 2.317275\n",
      "0.11\n",
      "step 700, entropy loss: 2.296310, l2_loss: 277.238495, total loss: 2.315717\n",
      "0.21\n",
      "step 800, entropy loss: 2.290182, l2_loss: 277.229675, total loss: 2.309588\n",
      "0.16\n",
      "step 900, entropy loss: 2.297042, l2_loss: 277.229736, total loss: 2.316448\n",
      "0.14\n",
      "step 1000, entropy loss: 2.295180, l2_loss: 277.231567, total loss: 2.314586\n",
      "0.19\n",
      "0.2116\n",
      "step 1100, entropy loss: 2.276883, l2_loss: 277.245331, total loss: 2.296290\n",
      "0.27\n",
      "step 1200, entropy loss: 2.291412, l2_loss: 277.263123, total loss: 2.310821\n",
      "0.21\n",
      "step 1300, entropy loss: 2.270262, l2_loss: 277.290649, total loss: 2.289672\n",
      "0.28\n",
      "step 1400, entropy loss: 2.282758, l2_loss: 277.329285, total loss: 2.302171\n",
      "0.15\n",
      "step 1500, entropy loss: 2.272206, l2_loss: 277.370056, total loss: 2.291621\n",
      "0.23\n",
      "step 1600, entropy loss: 2.282068, l2_loss: 277.414490, total loss: 2.301487\n",
      "0.19\n",
      "step 1700, entropy loss: 2.265751, l2_loss: 277.472961, total loss: 2.285174\n",
      "0.17\n",
      "step 1800, entropy loss: 2.268760, l2_loss: 277.544830, total loss: 2.288188\n",
      "0.22\n",
      "step 1900, entropy loss: 2.245128, l2_loss: 277.612488, total loss: 2.264561\n",
      "0.3\n",
      "step 2000, entropy loss: 2.254138, l2_loss: 277.700623, total loss: 2.273576\n",
      "0.2\n",
      "0.2116\n",
      "step 2100, entropy loss: 2.235850, l2_loss: 277.793762, total loss: 2.255296\n",
      "0.19\n",
      "step 2200, entropy loss: 2.204309, l2_loss: 277.889679, total loss: 2.223761\n",
      "0.28\n",
      "step 2300, entropy loss: 2.245336, l2_loss: 278.000916, total loss: 2.264796\n",
      "0.22\n",
      "step 2400, entropy loss: 2.221927, l2_loss: 278.128235, total loss: 2.241396\n",
      "0.22\n",
      "step 2500, entropy loss: 2.212276, l2_loss: 278.266632, total loss: 2.231754\n",
      "0.23\n",
      "step 2600, entropy loss: 2.223487, l2_loss: 278.408508, total loss: 2.242976\n",
      "0.15\n",
      "step 2700, entropy loss: 2.214169, l2_loss: 278.573151, total loss: 2.233669\n",
      "0.22\n",
      "step 2800, entropy loss: 2.172468, l2_loss: 278.740082, total loss: 2.191979\n",
      "0.21\n",
      "step 2900, entropy loss: 2.209046, l2_loss: 278.911041, total loss: 2.228570\n",
      "0.22\n",
      "step 3000, entropy loss: 2.196404, l2_loss: 279.123901, total loss: 2.215943\n",
      "0.2\n",
      "0.2315\n",
      "step 3100, entropy loss: 2.183414, l2_loss: 279.324738, total loss: 2.202966\n",
      "0.11\n",
      "step 3200, entropy loss: 2.147864, l2_loss: 279.564911, total loss: 2.167434\n",
      "0.2\n",
      "step 3300, entropy loss: 2.144369, l2_loss: 279.812073, total loss: 2.163955\n",
      "0.21\n",
      "step 3400, entropy loss: 2.084763, l2_loss: 280.075378, total loss: 2.104368\n",
      "0.32\n",
      "step 3500, entropy loss: 2.171738, l2_loss: 280.371216, total loss: 2.191364\n",
      "0.18\n",
      "step 3600, entropy loss: 2.121827, l2_loss: 280.649597, total loss: 2.141473\n",
      "0.25\n",
      "step 3700, entropy loss: 2.108814, l2_loss: 280.968567, total loss: 2.128482\n",
      "0.26\n",
      "step 3800, entropy loss: 2.098126, l2_loss: 281.306641, total loss: 2.117818\n",
      "0.25\n",
      "step 3900, entropy loss: 2.101762, l2_loss: 281.656189, total loss: 2.121478\n",
      "0.24\n",
      "step 4000, entropy loss: 2.021640, l2_loss: 282.038086, total loss: 2.041383\n",
      "0.29\n",
      "0.2669\n",
      "step 4100, entropy loss: 2.016110, l2_loss: 282.439880, total loss: 2.035881\n",
      "0.23\n",
      "step 4200, entropy loss: 1.962017, l2_loss: 282.850433, total loss: 1.981817\n",
      "0.28\n",
      "step 4300, entropy loss: 2.056940, l2_loss: 283.291565, total loss: 2.076771\n",
      "0.27\n",
      "step 4400, entropy loss: 2.042016, l2_loss: 283.730347, total loss: 2.061877\n",
      "0.24\n",
      "step 4500, entropy loss: 1.993724, l2_loss: 284.212952, total loss: 2.013619\n",
      "0.29\n",
      "step 4600, entropy loss: 1.933562, l2_loss: 284.699097, total loss: 1.953491\n",
      "0.31\n",
      "step 4700, entropy loss: 1.950263, l2_loss: 285.213531, total loss: 1.970228\n",
      "0.31\n",
      "step 4800, entropy loss: 1.837217, l2_loss: 285.758698, total loss: 1.857220\n",
      "0.36\n",
      "step 4900, entropy loss: 1.926076, l2_loss: 286.304565, total loss: 1.946117\n",
      "0.33\n",
      "step 5000, entropy loss: 1.997269, l2_loss: 286.875519, total loss: 2.017351\n",
      "0.27\n",
      "0.3141\n",
      "step 5100, entropy loss: 1.860994, l2_loss: 287.449951, total loss: 1.881115\n",
      "0.32\n",
      "step 5200, entropy loss: 1.838733, l2_loss: 288.042114, total loss: 1.858896\n",
      "0.29\n",
      "step 5300, entropy loss: 1.720232, l2_loss: 288.652679, total loss: 1.740438\n",
      "0.28\n",
      "step 5400, entropy loss: 1.913178, l2_loss: 289.258087, total loss: 1.933426\n",
      "0.29\n",
      "step 5500, entropy loss: 1.958849, l2_loss: 289.862305, total loss: 1.979140\n",
      "0.26\n",
      "step 5600, entropy loss: 1.684808, l2_loss: 290.455322, total loss: 1.705140\n",
      "0.34\n",
      "step 5700, entropy loss: 1.694997, l2_loss: 291.062042, total loss: 1.715371\n",
      "0.39\n",
      "step 5800, entropy loss: 1.792378, l2_loss: 291.643066, total loss: 1.812793\n",
      "0.35\n",
      "step 5900, entropy loss: 1.809823, l2_loss: 292.192749, total loss: 1.830276\n",
      "0.36\n",
      "step 6000, entropy loss: 1.733290, l2_loss: 292.744110, total loss: 1.753782\n",
      "0.33\n",
      "0.3454\n",
      "step 6100, entropy loss: 1.883829, l2_loss: 293.284088, total loss: 1.904359\n",
      "0.3\n",
      "step 6200, entropy loss: 1.623629, l2_loss: 293.798676, total loss: 1.644195\n",
      "0.37\n",
      "step 6300, entropy loss: 1.873089, l2_loss: 294.279846, total loss: 1.893688\n",
      "0.37\n",
      "step 6400, entropy loss: 1.640185, l2_loss: 294.719513, total loss: 1.660815\n",
      "0.41\n",
      "step 6500, entropy loss: 1.719466, l2_loss: 295.164337, total loss: 1.740128\n",
      "0.35\n",
      "step 6600, entropy loss: 1.641555, l2_loss: 295.600037, total loss: 1.662247\n",
      "0.36\n",
      "step 6700, entropy loss: 1.433000, l2_loss: 295.991730, total loss: 1.453719\n",
      "0.45\n",
      "step 6800, entropy loss: 1.606746, l2_loss: 296.413147, total loss: 1.627495\n",
      "0.33\n",
      "step 6900, entropy loss: 1.637933, l2_loss: 296.824738, total loss: 1.658711\n",
      "0.35\n",
      "step 7000, entropy loss: 1.527038, l2_loss: 297.222107, total loss: 1.547844\n",
      "0.47\n",
      "0.3843\n",
      "step 7100, entropy loss: 1.607128, l2_loss: 297.664734, total loss: 1.627964\n",
      "0.43\n",
      "step 7200, entropy loss: 1.588431, l2_loss: 298.075623, total loss: 1.609297\n",
      "0.4\n",
      "step 7300, entropy loss: 1.585912, l2_loss: 298.476715, total loss: 1.606806\n",
      "0.35\n",
      "step 7400, entropy loss: 1.540315, l2_loss: 298.854858, total loss: 1.561235\n",
      "0.38\n",
      "step 7500, entropy loss: 1.487761, l2_loss: 299.234436, total loss: 1.508707\n",
      "0.43\n",
      "step 7600, entropy loss: 1.456302, l2_loss: 299.639160, total loss: 1.477277\n",
      "0.48\n",
      "step 7700, entropy loss: 1.427941, l2_loss: 300.060913, total loss: 1.448946\n",
      "0.49\n",
      "step 7800, entropy loss: 1.570653, l2_loss: 300.478790, total loss: 1.591687\n",
      "0.43\n",
      "step 7900, entropy loss: 1.476181, l2_loss: 300.929199, total loss: 1.497246\n",
      "0.5\n",
      "step 8000, entropy loss: 1.577313, l2_loss: 301.382812, total loss: 1.598410\n",
      "0.42\n",
      "0.4615\n",
      "step 8100, entropy loss: 1.390169, l2_loss: 301.861206, total loss: 1.411299\n",
      "0.44\n",
      "step 8200, entropy loss: 1.425788, l2_loss: 302.362122, total loss: 1.446954\n",
      "0.44\n",
      "step 8300, entropy loss: 1.504917, l2_loss: 302.872375, total loss: 1.526118\n",
      "0.49\n",
      "step 8400, entropy loss: 1.365465, l2_loss: 303.382874, total loss: 1.386702\n",
      "0.5\n",
      "step 8500, entropy loss: 1.354714, l2_loss: 303.922913, total loss: 1.375988\n",
      "0.57\n",
      "step 8600, entropy loss: 1.467537, l2_loss: 304.505188, total loss: 1.488852\n",
      "0.51\n",
      "step 8700, entropy loss: 1.347801, l2_loss: 305.105896, total loss: 1.369158\n",
      "0.56\n",
      "step 8800, entropy loss: 1.599262, l2_loss: 305.715088, total loss: 1.620662\n",
      "0.48\n",
      "step 8900, entropy loss: 1.419299, l2_loss: 306.358521, total loss: 1.440744\n",
      "0.46\n",
      "step 9000, entropy loss: 1.347692, l2_loss: 307.057251, total loss: 1.369186\n",
      "0.45\n",
      "0.5375\n",
      "step 9100, entropy loss: 1.588127, l2_loss: 307.682678, total loss: 1.609665\n",
      "0.49\n",
      "step 9200, entropy loss: 1.298867, l2_loss: 308.352570, total loss: 1.320451\n",
      "0.48\n",
      "step 9300, entropy loss: 1.259289, l2_loss: 309.034882, total loss: 1.280921\n",
      "0.58\n",
      "step 9400, entropy loss: 1.372565, l2_loss: 309.766388, total loss: 1.394248\n",
      "0.54\n",
      "step 9500, entropy loss: 1.280773, l2_loss: 310.501343, total loss: 1.302508\n",
      "0.55\n",
      "step 9600, entropy loss: 1.406307, l2_loss: 311.205078, total loss: 1.428091\n",
      "0.49\n",
      "step 9700, entropy loss: 1.274396, l2_loss: 311.931030, total loss: 1.296231\n",
      "0.57\n",
      "step 9800, entropy loss: 1.230852, l2_loss: 312.662231, total loss: 1.252739\n",
      "0.51\n",
      "step 9900, entropy loss: 1.258255, l2_loss: 313.352295, total loss: 1.280190\n",
      "0.6\n",
      "step 10000, entropy loss: 1.019548, l2_loss: 314.079163, total loss: 1.041534\n",
      "0.59\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.5823\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [3,3],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [3,3],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 2.将卷积核conv1的filter数量从32-->64，conv2的filter数量从64-->128"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 4,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.302583, l2_loss: 444.122925, total loss: 2.333671\n",
      "0.2\n",
      "step 200, entropy loss: 2.299436, l2_loss: 444.067352, total loss: 2.330520\n",
      "0.18\n",
      "step 300, entropy loss: 2.294039, l2_loss: 444.015808, total loss: 2.325121\n",
      "0.18\n",
      "step 400, entropy loss: 2.297845, l2_loss: 443.969299, total loss: 2.328923\n",
      "0.17\n",
      "step 500, entropy loss: 2.295368, l2_loss: 443.926727, total loss: 2.326443\n",
      "0.19\n",
      "step 600, entropy loss: 2.294392, l2_loss: 443.890381, total loss: 2.325464\n",
      "0.19\n",
      "step 700, entropy loss: 2.295955, l2_loss: 443.860077, total loss: 2.327025\n",
      "0.18\n",
      "step 800, entropy loss: 2.289993, l2_loss: 443.836365, total loss: 2.321062\n",
      "0.23\n",
      "step 900, entropy loss: 2.297581, l2_loss: 443.821045, total loss: 2.328648\n",
      "0.18\n",
      "step 1000, entropy loss: 2.287589, l2_loss: 443.815125, total loss: 2.318656\n",
      "0.19\n",
      "0.2135\n",
      "step 1100, entropy loss: 2.272031, l2_loss: 443.819031, total loss: 2.303098\n",
      "0.21\n",
      "step 1200, entropy loss: 2.272752, l2_loss: 443.839752, total loss: 2.303821\n",
      "0.19\n",
      "step 1300, entropy loss: 2.260355, l2_loss: 443.865601, total loss: 2.291426\n",
      "0.26\n",
      "step 1400, entropy loss: 2.268757, l2_loss: 443.902252, total loss: 2.299830\n",
      "0.25\n",
      "step 1500, entropy loss: 2.253609, l2_loss: 443.960907, total loss: 2.284686\n",
      "0.22\n",
      "step 1600, entropy loss: 2.246959, l2_loss: 444.019165, total loss: 2.278040\n",
      "0.2\n",
      "step 1700, entropy loss: 2.241751, l2_loss: 444.092896, total loss: 2.272837\n",
      "0.19\n",
      "step 1800, entropy loss: 2.248196, l2_loss: 444.188110, total loss: 2.279289\n",
      "0.21\n",
      "step 1900, entropy loss: 2.197183, l2_loss: 444.288025, total loss: 2.228283\n",
      "0.22\n",
      "step 2000, entropy loss: 2.203037, l2_loss: 444.412292, total loss: 2.234146\n",
      "0.29\n",
      "0.2267\n",
      "step 2100, entropy loss: 2.179825, l2_loss: 444.558655, total loss: 2.210944\n",
      "0.24\n",
      "step 2200, entropy loss: 2.196031, l2_loss: 444.738342, total loss: 2.227163\n",
      "0.24\n",
      "step 2300, entropy loss: 2.199130, l2_loss: 444.954315, total loss: 2.230277\n",
      "0.16\n",
      "step 2400, entropy loss: 2.160368, l2_loss: 445.196625, total loss: 2.191532\n",
      "0.21\n",
      "step 2500, entropy loss: 2.124995, l2_loss: 445.458679, total loss: 2.156178\n",
      "0.25\n",
      "step 2600, entropy loss: 2.160391, l2_loss: 445.764191, total loss: 2.191595\n",
      "0.2\n",
      "step 2700, entropy loss: 2.088344, l2_loss: 446.108765, total loss: 2.119571\n",
      "0.28\n",
      "step 2800, entropy loss: 2.007863, l2_loss: 446.505890, total loss: 2.039119\n",
      "0.24\n",
      "step 2900, entropy loss: 1.976974, l2_loss: 446.902771, total loss: 2.008258\n",
      "0.26\n",
      "step 3000, entropy loss: 2.010336, l2_loss: 447.339600, total loss: 2.041650\n",
      "0.24\n",
      "0.2629\n",
      "step 3100, entropy loss: 1.986766, l2_loss: 447.795471, total loss: 2.018111\n",
      "0.22\n",
      "step 3200, entropy loss: 1.961093, l2_loss: 448.267883, total loss: 1.992472\n",
      "0.25\n",
      "step 3300, entropy loss: 1.847367, l2_loss: 448.747437, total loss: 1.878779\n",
      "0.32\n",
      "step 3400, entropy loss: 1.838304, l2_loss: 449.279816, total loss: 1.869754\n",
      "0.28\n",
      "step 3500, entropy loss: 1.836460, l2_loss: 449.826111, total loss: 1.867948\n",
      "0.31\n",
      "step 3600, entropy loss: 1.881299, l2_loss: 450.389038, total loss: 1.912826\n",
      "0.34\n",
      "step 3700, entropy loss: 1.794506, l2_loss: 451.044373, total loss: 1.826079\n",
      "0.34\n",
      "step 3800, entropy loss: 1.638885, l2_loss: 451.747925, total loss: 1.670507\n",
      "0.46\n",
      "step 3900, entropy loss: 1.635145, l2_loss: 452.507538, total loss: 1.666821\n",
      "0.49\n",
      "step 4000, entropy loss: 1.650622, l2_loss: 453.291626, total loss: 1.682352\n",
      "0.4\n",
      "0.4081\n",
      "step 4100, entropy loss: 1.493529, l2_loss: 454.137390, total loss: 1.525318\n",
      "0.49\n",
      "step 4200, entropy loss: 1.521134, l2_loss: 454.979980, total loss: 1.552982\n",
      "0.51\n",
      "step 4300, entropy loss: 1.573557, l2_loss: 455.828156, total loss: 1.605465\n",
      "0.51\n",
      "step 4400, entropy loss: 1.594118, l2_loss: 456.683289, total loss: 1.626086\n",
      "0.5\n",
      "step 4500, entropy loss: 1.376136, l2_loss: 457.536377, total loss: 1.408164\n",
      "0.53\n",
      "step 4600, entropy loss: 1.355855, l2_loss: 458.445496, total loss: 1.387946\n",
      "0.62\n",
      "step 4700, entropy loss: 1.354990, l2_loss: 459.387207, total loss: 1.387147\n",
      "0.53\n",
      "step 4800, entropy loss: 1.282025, l2_loss: 460.370667, total loss: 1.314251\n",
      "0.55\n",
      "step 4900, entropy loss: 1.317550, l2_loss: 461.408325, total loss: 1.349849\n",
      "0.52\n",
      "step 5000, entropy loss: 1.345545, l2_loss: 462.487976, total loss: 1.377919\n",
      "0.63\n",
      "0.6055\n",
      "step 5100, entropy loss: 1.265296, l2_loss: 463.582916, total loss: 1.297747\n",
      "0.58\n",
      "step 5200, entropy loss: 1.148970, l2_loss: 464.647766, total loss: 1.181495\n",
      "0.63\n",
      "step 5300, entropy loss: 0.925836, l2_loss: 465.714539, total loss: 0.958436\n",
      "0.69\n",
      "step 5400, entropy loss: 0.985745, l2_loss: 466.908264, total loss: 1.018429\n",
      "0.7\n",
      "step 5500, entropy loss: 0.924609, l2_loss: 468.031555, total loss: 0.957372\n",
      "0.73\n",
      "step 5600, entropy loss: 0.851903, l2_loss: 469.130615, total loss: 0.884742\n",
      "0.72\n",
      "step 5700, entropy loss: 0.799308, l2_loss: 470.205536, total loss: 0.832223\n",
      "0.75\n",
      "step 5800, entropy loss: 0.771168, l2_loss: 471.271606, total loss: 0.804157\n",
      "0.8\n",
      "step 5900, entropy loss: 0.846665, l2_loss: 472.315125, total loss: 0.879727\n",
      "0.75\n",
      "step 6000, entropy loss: 0.757560, l2_loss: 473.320129, total loss: 0.790693\n",
      "0.86\n",
      "0.772\n",
      "step 6100, entropy loss: 0.932270, l2_loss: 474.364380, total loss: 0.965476\n",
      "0.75\n",
      "step 6200, entropy loss: 0.901765, l2_loss: 475.337067, total loss: 0.935039\n",
      "0.66\n",
      "step 6300, entropy loss: 0.864221, l2_loss: 476.330444, total loss: 0.897565\n",
      "0.74\n",
      "step 6400, entropy loss: 0.655395, l2_loss: 477.270569, total loss: 0.688804\n",
      "0.78\n",
      "step 6500, entropy loss: 0.806643, l2_loss: 478.179016, total loss: 0.840116\n",
      "0.73\n",
      "step 6600, entropy loss: 0.570982, l2_loss: 479.053650, total loss: 0.604516\n",
      "0.84\n",
      "step 6700, entropy loss: 0.433917, l2_loss: 479.937378, total loss: 0.467513\n",
      "0.89\n",
      "step 6800, entropy loss: 0.587417, l2_loss: 480.806274, total loss: 0.621074\n",
      "0.85\n",
      "step 6900, entropy loss: 0.708899, l2_loss: 481.616699, total loss: 0.742612\n",
      "0.84\n",
      "step 7000, entropy loss: 0.873523, l2_loss: 482.490234, total loss: 0.907297\n",
      "0.8\n",
      "0.8382\n",
      "step 7100, entropy loss: 0.656880, l2_loss: 483.240417, total loss: 0.690707\n",
      "0.81\n",
      "step 7200, entropy loss: 0.633616, l2_loss: 484.021545, total loss: 0.667497\n",
      "0.85\n",
      "step 7300, entropy loss: 0.595018, l2_loss: 484.789307, total loss: 0.628953\n",
      "0.84\n",
      "step 7400, entropy loss: 0.426624, l2_loss: 485.478394, total loss: 0.460608\n",
      "0.87\n",
      "step 7500, entropy loss: 0.536955, l2_loss: 486.233337, total loss: 0.570991\n",
      "0.87\n",
      "step 7600, entropy loss: 0.494953, l2_loss: 486.892761, total loss: 0.529036\n",
      "0.87\n",
      "step 7700, entropy loss: 0.426642, l2_loss: 487.557404, total loss: 0.460771\n",
      "0.86\n",
      "step 7800, entropy loss: 0.697541, l2_loss: 488.258636, total loss: 0.731719\n",
      "0.81\n",
      "step 7900, entropy loss: 0.364663, l2_loss: 488.872437, total loss: 0.398884\n",
      "0.91\n",
      "step 8000, entropy loss: 0.449238, l2_loss: 489.490234, total loss: 0.483503\n",
      "0.9\n",
      "0.8793\n",
      "step 8100, entropy loss: 0.488851, l2_loss: 490.052795, total loss: 0.523155\n",
      "0.86\n",
      "step 8200, entropy loss: 0.481221, l2_loss: 490.643982, total loss: 0.515567\n",
      "0.87\n",
      "step 8300, entropy loss: 0.434775, l2_loss: 491.196228, total loss: 0.469159\n",
      "0.9\n",
      "step 8400, entropy loss: 0.451958, l2_loss: 491.805237, total loss: 0.486384\n",
      "0.88\n",
      "step 8500, entropy loss: 0.443544, l2_loss: 492.342621, total loss: 0.478008\n",
      "0.91\n",
      "step 8600, entropy loss: 0.457967, l2_loss: 492.860931, total loss: 0.492467\n",
      "0.95\n",
      "step 8700, entropy loss: 0.468204, l2_loss: 493.393341, total loss: 0.502741\n",
      "0.89\n",
      "step 8800, entropy loss: 0.404163, l2_loss: 493.884918, total loss: 0.438735\n",
      "0.91\n",
      "step 8900, entropy loss: 0.361955, l2_loss: 494.336670, total loss: 0.396559\n",
      "0.92\n",
      "step 9000, entropy loss: 0.587531, l2_loss: 494.849365, total loss: 0.622171\n",
      "0.84\n",
      "0.8916\n",
      "step 9100, entropy loss: 0.398793, l2_loss: 495.336975, total loss: 0.433466\n",
      "0.91\n",
      "step 9200, entropy loss: 0.248172, l2_loss: 495.826294, total loss: 0.282880\n",
      "0.95\n",
      "step 9300, entropy loss: 0.390771, l2_loss: 496.222595, total loss: 0.425507\n",
      "0.9\n",
      "step 9400, entropy loss: 0.450034, l2_loss: 496.643311, total loss: 0.484799\n",
      "0.87\n",
      "step 9500, entropy loss: 0.349161, l2_loss: 497.096161, total loss: 0.383957\n",
      "0.89\n",
      "step 9600, entropy loss: 0.354897, l2_loss: 497.520660, total loss: 0.389724\n",
      "0.89\n",
      "step 9700, entropy loss: 0.286530, l2_loss: 497.934448, total loss: 0.321386\n",
      "0.93\n",
      "step 9800, entropy loss: 0.250911, l2_loss: 498.354797, total loss: 0.285796\n",
      "0.93\n",
      "step 9900, entropy loss: 0.342316, l2_loss: 498.751587, total loss: 0.377229\n",
      "0.94\n",
      "step 10000, entropy loss: 0.391499, l2_loss: 499.133667, total loss: 0.426439\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.91\n",
      "0.8994\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 128, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 3.learning_rate 从0.01-->0.001"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 5,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.299752, l2_loss: 536.363403, total loss: 2.337297\n",
      "0.1\n",
      "step 200, entropy loss: 2.304395, l2_loss: 536.354126, total loss: 2.341940\n",
      "0.08\n",
      "step 300, entropy loss: 2.304717, l2_loss: 536.344788, total loss: 2.342261\n",
      "0.09\n",
      "step 400, entropy loss: 2.302081, l2_loss: 536.335693, total loss: 2.339624\n",
      "0.09\n",
      "step 500, entropy loss: 2.301682, l2_loss: 536.326660, total loss: 2.339225\n",
      "0.07\n",
      "step 600, entropy loss: 2.302359, l2_loss: 536.317505, total loss: 2.339901\n",
      "0.07\n",
      "step 700, entropy loss: 2.302302, l2_loss: 536.308655, total loss: 2.339844\n",
      "0.12\n",
      "step 800, entropy loss: 2.302186, l2_loss: 536.299622, total loss: 2.339727\n",
      "0.1\n",
      "step 900, entropy loss: 2.301422, l2_loss: 536.290894, total loss: 2.338962\n",
      "0.12\n",
      "step 1000, entropy loss: 2.298276, l2_loss: 536.282288, total loss: 2.335815\n",
      "0.11\n",
      "0.1283\n",
      "step 1100, entropy loss: 2.300922, l2_loss: 536.273560, total loss: 2.338462\n",
      "0.14\n",
      "step 1200, entropy loss: 2.300061, l2_loss: 536.264709, total loss: 2.337600\n",
      "0.12\n",
      "step 1300, entropy loss: 2.297343, l2_loss: 536.255737, total loss: 2.334881\n",
      "0.13\n",
      "step 1400, entropy loss: 2.299050, l2_loss: 536.247131, total loss: 2.336588\n",
      "0.13\n",
      "step 1500, entropy loss: 2.300539, l2_loss: 536.238342, total loss: 2.338075\n",
      "0.14\n",
      "step 1600, entropy loss: 2.297730, l2_loss: 536.229797, total loss: 2.335266\n",
      "0.13\n",
      "step 1700, entropy loss: 2.299777, l2_loss: 536.221252, total loss: 2.337313\n",
      "0.15\n",
      "step 1800, entropy loss: 2.300205, l2_loss: 536.212891, total loss: 2.337740\n",
      "0.15\n",
      "step 1900, entropy loss: 2.304156, l2_loss: 536.204346, total loss: 2.341691\n",
      "0.1\n",
      "step 2000, entropy loss: 2.296402, l2_loss: 536.195801, total loss: 2.333936\n",
      "0.16\n",
      "0.1682\n",
      "step 2100, entropy loss: 2.298557, l2_loss: 536.187317, total loss: 2.336090\n",
      "0.13\n",
      "step 2200, entropy loss: 2.299808, l2_loss: 536.179138, total loss: 2.337341\n",
      "0.17\n",
      "step 2300, entropy loss: 2.305494, l2_loss: 536.170715, total loss: 2.343026\n",
      "0.16\n",
      "step 2400, entropy loss: 2.301384, l2_loss: 536.162415, total loss: 2.338916\n",
      "0.17\n",
      "step 2500, entropy loss: 2.295768, l2_loss: 536.154175, total loss: 2.333299\n",
      "0.1\n",
      "step 2600, entropy loss: 2.296093, l2_loss: 536.146057, total loss: 2.333623\n",
      "0.18\n",
      "step 2700, entropy loss: 2.298214, l2_loss: 536.137939, total loss: 2.335744\n",
      "0.14\n",
      "step 2800, entropy loss: 2.299397, l2_loss: 536.130005, total loss: 2.336926\n",
      "0.15\n",
      "step 2900, entropy loss: 2.302294, l2_loss: 536.121948, total loss: 2.339822\n",
      "0.14\n",
      "step 3000, entropy loss: 2.300287, l2_loss: 536.114075, total loss: 2.337815\n",
      "0.19\n",
      "0.1809\n",
      "step 3100, entropy loss: 2.293998, l2_loss: 536.106384, total loss: 2.331525\n",
      "0.16\n",
      "step 3200, entropy loss: 2.295510, l2_loss: 536.098450, total loss: 2.333037\n",
      "0.21\n",
      "step 3300, entropy loss: 2.301989, l2_loss: 536.090393, total loss: 2.339515\n",
      "0.14\n",
      "step 3400, entropy loss: 2.296512, l2_loss: 536.082458, total loss: 2.334038\n",
      "0.23\n",
      "step 3500, entropy loss: 2.296666, l2_loss: 536.074768, total loss: 2.334191\n",
      "0.19\n",
      "step 3600, entropy loss: 2.298598, l2_loss: 536.067139, total loss: 2.336123\n",
      "0.21\n",
      "step 3700, entropy loss: 2.295787, l2_loss: 536.059692, total loss: 2.333311\n",
      "0.24\n",
      "step 3800, entropy loss: 2.298376, l2_loss: 536.052124, total loss: 2.335900\n",
      "0.19\n",
      "step 3900, entropy loss: 2.298261, l2_loss: 536.044556, total loss: 2.335784\n",
      "0.16\n",
      "step 4000, entropy loss: 2.297637, l2_loss: 536.037292, total loss: 2.335160\n",
      "0.12\n",
      "0.19\n",
      "step 4100, entropy loss: 2.293754, l2_loss: 536.029907, total loss: 2.331276\n",
      "0.24\n",
      "step 4200, entropy loss: 2.297000, l2_loss: 536.022339, total loss: 2.334522\n",
      "0.09\n",
      "step 4300, entropy loss: 2.301424, l2_loss: 536.015198, total loss: 2.338945\n",
      "0.13\n",
      "step 4400, entropy loss: 2.298364, l2_loss: 536.007812, total loss: 2.335884\n",
      "0.16\n",
      "step 4500, entropy loss: 2.292919, l2_loss: 536.000732, total loss: 2.330439\n",
      "0.19\n",
      "step 4600, entropy loss: 2.294682, l2_loss: 535.993774, total loss: 2.332202\n",
      "0.18\n",
      "step 4700, entropy loss: 2.290233, l2_loss: 535.986755, total loss: 2.327752\n",
      "0.2\n",
      "step 4800, entropy loss: 2.299138, l2_loss: 535.979858, total loss: 2.336657\n",
      "0.2\n",
      "step 4900, entropy loss: 2.298298, l2_loss: 535.972961, total loss: 2.335816\n",
      "0.12\n",
      "step 5000, entropy loss: 2.291180, l2_loss: 535.966248, total loss: 2.328697\n",
      "0.2\n",
      "0.1993\n",
      "step 5100, entropy loss: 2.288370, l2_loss: 535.959473, total loss: 2.325887\n",
      "0.21\n",
      "step 5200, entropy loss: 2.295917, l2_loss: 535.952942, total loss: 2.333434\n",
      "0.14\n",
      "step 5300, entropy loss: 2.298162, l2_loss: 535.946289, total loss: 2.335679\n",
      "0.13\n",
      "step 5400, entropy loss: 2.290872, l2_loss: 535.939636, total loss: 2.328387\n",
      "0.22\n",
      "step 5500, entropy loss: 2.288040, l2_loss: 535.933105, total loss: 2.325556\n",
      "0.23\n",
      "step 5600, entropy loss: 2.294844, l2_loss: 535.926514, total loss: 2.332359\n",
      "0.18\n",
      "step 5700, entropy loss: 2.295137, l2_loss: 535.920166, total loss: 2.332651\n",
      "0.19\n",
      "step 5800, entropy loss: 2.293116, l2_loss: 535.913940, total loss: 2.330630\n",
      "0.21\n",
      "step 5900, entropy loss: 2.289854, l2_loss: 535.907288, total loss: 2.327368\n",
      "0.21\n",
      "step 6000, entropy loss: 2.296787, l2_loss: 535.900818, total loss: 2.334300\n",
      "0.14\n",
      "0.2016\n",
      "step 6100, entropy loss: 2.289736, l2_loss: 535.894653, total loss: 2.327248\n",
      "0.21\n",
      "step 6200, entropy loss: 2.289929, l2_loss: 535.888672, total loss: 2.327441\n",
      "0.2\n",
      "step 6300, entropy loss: 2.285891, l2_loss: 535.882690, total loss: 2.323403\n",
      "0.23\n",
      "step 6400, entropy loss: 2.291268, l2_loss: 535.876770, total loss: 2.328779\n",
      "0.19\n",
      "step 6500, entropy loss: 2.292975, l2_loss: 535.870605, total loss: 2.330486\n",
      "0.14\n",
      "step 6600, entropy loss: 2.289687, l2_loss: 535.865051, total loss: 2.327198\n",
      "0.2\n",
      "step 6700, entropy loss: 2.295700, l2_loss: 535.859131, total loss: 2.333210\n",
      "0.18\n",
      "step 6800, entropy loss: 2.297819, l2_loss: 535.853455, total loss: 2.335329\n",
      "0.13\n",
      "step 6900, entropy loss: 2.292873, l2_loss: 535.848083, total loss: 2.330383\n",
      "0.2\n",
      "step 7000, entropy loss: 2.294617, l2_loss: 535.842651, total loss: 2.332126\n",
      "0.19\n",
      "0.1984\n",
      "step 7100, entropy loss: 2.288153, l2_loss: 535.837158, total loss: 2.325661\n",
      "0.26\n",
      "step 7200, entropy loss: 2.283491, l2_loss: 535.832031, total loss: 2.320999\n",
      "0.19\n",
      "step 7300, entropy loss: 2.289763, l2_loss: 535.826721, total loss: 2.327271\n",
      "0.18\n",
      "step 7400, entropy loss: 2.292715, l2_loss: 535.821472, total loss: 2.330223\n",
      "0.17\n",
      "step 7500, entropy loss: 2.286473, l2_loss: 535.816406, total loss: 2.323980\n",
      "0.2\n",
      "step 7600, entropy loss: 2.297152, l2_loss: 535.811523, total loss: 2.334659\n",
      "0.14\n",
      "step 7700, entropy loss: 2.284505, l2_loss: 535.806641, total loss: 2.322012\n",
      "0.22\n",
      "step 7800, entropy loss: 2.283659, l2_loss: 535.801941, total loss: 2.321165\n",
      "0.24\n",
      "step 7900, entropy loss: 2.282960, l2_loss: 535.797363, total loss: 2.320466\n",
      "0.17\n",
      "step 8000, entropy loss: 2.288920, l2_loss: 535.792847, total loss: 2.326426\n",
      "0.18\n",
      "0.1999\n",
      "step 8100, entropy loss: 2.281766, l2_loss: 535.788391, total loss: 2.319271\n",
      "0.19\n",
      "step 8200, entropy loss: 2.287984, l2_loss: 535.784424, total loss: 2.325489\n",
      "0.19\n",
      "step 8300, entropy loss: 2.276031, l2_loss: 535.780273, total loss: 2.313535\n",
      "0.29\n",
      "step 8400, entropy loss: 2.279198, l2_loss: 535.775452, total loss: 2.316702\n",
      "0.22\n",
      "step 8500, entropy loss: 2.286617, l2_loss: 535.770935, total loss: 2.324121\n",
      "0.18\n",
      "step 8600, entropy loss: 2.288245, l2_loss: 535.766663, total loss: 2.325749\n",
      "0.18\n",
      "step 8700, entropy loss: 2.285096, l2_loss: 535.762939, total loss: 2.322600\n",
      "0.17\n",
      "step 8800, entropy loss: 2.274577, l2_loss: 535.759155, total loss: 2.312080\n",
      "0.21\n",
      "step 8900, entropy loss: 2.289526, l2_loss: 535.755310, total loss: 2.327028\n",
      "0.17\n",
      "step 9000, entropy loss: 2.282834, l2_loss: 535.751465, total loss: 2.320336\n",
      "0.14\n",
      "0.1965\n",
      "step 9100, entropy loss: 2.291096, l2_loss: 535.747925, total loss: 2.328598\n",
      "0.16\n",
      "step 9200, entropy loss: 2.287673, l2_loss: 535.744141, total loss: 2.325175\n",
      "0.19\n",
      "step 9300, entropy loss: 2.279128, l2_loss: 535.740723, total loss: 2.316630\n",
      "0.25\n",
      "step 9400, entropy loss: 2.278139, l2_loss: 535.738037, total loss: 2.315640\n",
      "0.19\n",
      "step 9500, entropy loss: 2.282651, l2_loss: 535.734497, total loss: 2.320152\n",
      "0.19\n",
      "step 9600, entropy loss: 2.277570, l2_loss: 535.731384, total loss: 2.315071\n",
      "0.22\n",
      "step 9700, entropy loss: 2.285197, l2_loss: 535.728455, total loss: 2.322698\n",
      "0.2\n",
      "step 9800, entropy loss: 2.269554, l2_loss: 535.725098, total loss: 2.307055\n",
      "0.22\n",
      "step 9900, entropy loss: 2.287352, l2_loss: 535.722168, total loss: 2.324852\n",
      "0.13\n",
      "step 10000, entropy loss: 2.278132, l2_loss: 535.720154, total loss: 2.315632\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.19\n",
      "0.1907\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.001\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))\n"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 4.将正则化因子从7e-5-->4e-5"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 6,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.298884, l2_loss: 629.301208, total loss: 2.324056\n",
      "0.1\n",
      "step 200, entropy loss: 2.299740, l2_loss: 629.252441, total loss: 2.324910\n",
      "0.19\n",
      "step 300, entropy loss: 2.301408, l2_loss: 629.208679, total loss: 2.326576\n",
      "0.15\n",
      "step 400, entropy loss: 2.293762, l2_loss: 629.169434, total loss: 2.318929\n",
      "0.18\n",
      "step 500, entropy loss: 2.298795, l2_loss: 629.135254, total loss: 2.323960\n",
      "0.19\n",
      "step 600, entropy loss: 2.292333, l2_loss: 629.105164, total loss: 2.317497\n",
      "0.14\n",
      "step 700, entropy loss: 2.282220, l2_loss: 629.081299, total loss: 2.307384\n",
      "0.21\n",
      "step 800, entropy loss: 2.299220, l2_loss: 629.064575, total loss: 2.324383\n",
      "0.1\n",
      "step 900, entropy loss: 2.303152, l2_loss: 629.055786, total loss: 2.328314\n",
      "0.18\n",
      "step 1000, entropy loss: 2.298232, l2_loss: 629.054688, total loss: 2.323394\n",
      "0.2\n",
      "0.2083\n",
      "step 1100, entropy loss: 2.282600, l2_loss: 629.068115, total loss: 2.307763\n",
      "0.17\n",
      "step 1200, entropy loss: 2.268034, l2_loss: 629.091614, total loss: 2.293198\n",
      "0.2\n",
      "step 1300, entropy loss: 2.262599, l2_loss: 629.121216, total loss: 2.287764\n",
      "0.24\n",
      "step 1400, entropy loss: 2.268775, l2_loss: 629.159424, total loss: 2.293941\n",
      "0.2\n",
      "step 1500, entropy loss: 2.235280, l2_loss: 629.200500, total loss: 2.260448\n",
      "0.25\n",
      "step 1600, entropy loss: 2.243849, l2_loss: 629.267029, total loss: 2.269019\n",
      "0.16\n",
      "step 1700, entropy loss: 2.217922, l2_loss: 629.342163, total loss: 2.243096\n",
      "0.21\n",
      "step 1800, entropy loss: 2.194305, l2_loss: 629.429626, total loss: 2.219483\n",
      "0.27\n",
      "step 1900, entropy loss: 2.242268, l2_loss: 629.532043, total loss: 2.267450\n",
      "0.13\n",
      "step 2000, entropy loss: 2.231686, l2_loss: 629.647827, total loss: 2.256872\n",
      "0.22\n",
      "0.2094\n",
      "step 2100, entropy loss: 2.208789, l2_loss: 629.765869, total loss: 2.233979\n",
      "0.21\n",
      "step 2200, entropy loss: 2.175279, l2_loss: 629.922791, total loss: 2.200476\n",
      "0.28\n",
      "step 2300, entropy loss: 2.124490, l2_loss: 630.097534, total loss: 2.149694\n",
      "0.32\n",
      "step 2400, entropy loss: 2.196390, l2_loss: 630.283203, total loss: 2.221601\n",
      "0.27\n",
      "step 2500, entropy loss: 2.147800, l2_loss: 630.511963, total loss: 2.173020\n",
      "0.18\n",
      "step 2600, entropy loss: 2.117860, l2_loss: 630.755737, total loss: 2.143090\n",
      "0.29\n",
      "step 2700, entropy loss: 2.148262, l2_loss: 631.027588, total loss: 2.173503\n",
      "0.19\n",
      "step 2800, entropy loss: 2.143210, l2_loss: 631.354370, total loss: 2.168464\n",
      "0.2\n",
      "step 2900, entropy loss: 2.096629, l2_loss: 631.671570, total loss: 2.121896\n",
      "0.24\n",
      "step 3000, entropy loss: 2.049663, l2_loss: 632.025513, total loss: 2.074944\n",
      "0.22\n",
      "0.2483\n",
      "step 3100, entropy loss: 2.045915, l2_loss: 632.424561, total loss: 2.071212\n",
      "0.26\n",
      "step 3200, entropy loss: 1.958003, l2_loss: 632.825684, total loss: 1.983316\n",
      "0.23\n",
      "step 3300, entropy loss: 1.867097, l2_loss: 633.251160, total loss: 1.892427\n",
      "0.34\n",
      "step 3400, entropy loss: 1.971296, l2_loss: 633.704468, total loss: 1.996644\n",
      "0.29\n",
      "step 3500, entropy loss: 2.013062, l2_loss: 634.155396, total loss: 2.038428\n",
      "0.31\n",
      "step 3600, entropy loss: 1.921187, l2_loss: 634.640869, total loss: 1.946573\n",
      "0.35\n",
      "step 3700, entropy loss: 1.965459, l2_loss: 635.155884, total loss: 1.990865\n",
      "0.33\n",
      "step 3800, entropy loss: 1.876759, l2_loss: 635.654785, total loss: 1.902185\n",
      "0.28\n",
      "step 3900, entropy loss: 1.765216, l2_loss: 636.237488, total loss: 1.790665\n",
      "0.34\n",
      "step 4000, entropy loss: 1.736211, l2_loss: 636.871765, total loss: 1.761686\n",
      "0.38\n",
      "0.3446\n",
      "step 4100, entropy loss: 1.770013, l2_loss: 637.547180, total loss: 1.795515\n",
      "0.37\n",
      "step 4200, entropy loss: 1.608183, l2_loss: 638.249207, total loss: 1.633713\n",
      "0.38\n",
      "step 4300, entropy loss: 1.616742, l2_loss: 638.958862, total loss: 1.642301\n",
      "0.46\n",
      "step 4400, entropy loss: 1.643440, l2_loss: 639.715942, total loss: 1.669029\n",
      "0.43\n",
      "step 4500, entropy loss: 1.647625, l2_loss: 640.461426, total loss: 1.673244\n",
      "0.36\n",
      "step 4600, entropy loss: 1.592890, l2_loss: 641.212708, total loss: 1.618538\n",
      "0.45\n",
      "step 4700, entropy loss: 1.729832, l2_loss: 641.965820, total loss: 1.755510\n",
      "0.37\n",
      "step 4800, entropy loss: 1.558414, l2_loss: 642.721680, total loss: 1.584123\n",
      "0.47\n",
      "step 4900, entropy loss: 1.392857, l2_loss: 643.494263, total loss: 1.418597\n",
      "0.44\n",
      "step 5000, entropy loss: 1.458175, l2_loss: 644.265137, total loss: 1.483945\n",
      "0.46\n",
      "0.5059\n",
      "step 5100, entropy loss: 1.405882, l2_loss: 645.093140, total loss: 1.431685\n",
      "0.56\n",
      "step 5200, entropy loss: 1.356389, l2_loss: 645.979919, total loss: 1.382228\n",
      "0.58\n",
      "step 5300, entropy loss: 1.378726, l2_loss: 646.878662, total loss: 1.404601\n",
      "0.57\n",
      "step 5400, entropy loss: 1.108899, l2_loss: 647.837158, total loss: 1.134813\n",
      "0.56\n",
      "step 5500, entropy loss: 1.260556, l2_loss: 648.861450, total loss: 1.286510\n",
      "0.6\n",
      "step 5600, entropy loss: 1.154483, l2_loss: 649.883606, total loss: 1.180478\n",
      "0.65\n",
      "step 5700, entropy loss: 1.156342, l2_loss: 650.971191, total loss: 1.182381\n",
      "0.62\n",
      "step 5800, entropy loss: 1.150194, l2_loss: 652.070923, total loss: 1.176277\n",
      "0.61\n",
      "step 5900, entropy loss: 0.936823, l2_loss: 653.172729, total loss: 0.962950\n",
      "0.76\n",
      "step 6000, entropy loss: 1.093848, l2_loss: 654.249268, total loss: 1.120018\n",
      "0.69\n",
      "0.6841\n",
      "step 6100, entropy loss: 1.020269, l2_loss: 655.310486, total loss: 1.046481\n",
      "0.62\n",
      "step 6200, entropy loss: 0.991004, l2_loss: 656.360168, total loss: 1.017259\n",
      "0.67\n",
      "step 6300, entropy loss: 0.670287, l2_loss: 657.399170, total loss: 0.696583\n",
      "0.84\n",
      "step 6400, entropy loss: 0.859855, l2_loss: 658.408203, total loss: 0.886191\n",
      "0.8\n",
      "step 6500, entropy loss: 0.839698, l2_loss: 659.395508, total loss: 0.866074\n",
      "0.75\n",
      "step 6600, entropy loss: 0.910121, l2_loss: 660.341858, total loss: 0.936535\n",
      "0.78\n",
      "step 6700, entropy loss: 0.914589, l2_loss: 661.311707, total loss: 0.941042\n",
      "0.75\n",
      "step 6800, entropy loss: 0.559882, l2_loss: 662.263672, total loss: 0.586373\n",
      "0.85\n",
      "step 6900, entropy loss: 0.827044, l2_loss: 663.177429, total loss: 0.853571\n",
      "0.75\n",
      "step 7000, entropy loss: 0.919397, l2_loss: 664.037842, total loss: 0.945959\n",
      "0.74\n",
      "0.7777\n",
      "step 7100, entropy loss: 0.809658, l2_loss: 664.864807, total loss: 0.836253\n",
      "0.8\n",
      "step 7200, entropy loss: 0.742141, l2_loss: 665.682739, total loss: 0.768768\n",
      "0.81\n",
      "step 7300, entropy loss: 0.602803, l2_loss: 666.531372, total loss: 0.629464\n",
      "0.87\n",
      "step 7400, entropy loss: 0.531834, l2_loss: 667.308350, total loss: 0.558526\n",
      "0.81\n",
      "step 7500, entropy loss: 0.634126, l2_loss: 668.035583, total loss: 0.660848\n",
      "0.76\n",
      "step 7600, entropy loss: 0.587653, l2_loss: 668.767273, total loss: 0.614404\n",
      "0.87\n",
      "step 7700, entropy loss: 0.489753, l2_loss: 669.471680, total loss: 0.516532\n",
      "0.85\n",
      "step 7800, entropy loss: 0.479465, l2_loss: 670.166931, total loss: 0.506271\n",
      "0.88\n",
      "step 7900, entropy loss: 0.565668, l2_loss: 670.838196, total loss: 0.592501\n",
      "0.81\n",
      "step 8000, entropy loss: 0.965214, l2_loss: 671.487671, total loss: 0.992073\n",
      "0.67\n",
      "0.5782\n",
      "step 8100, entropy loss: 0.482348, l2_loss: 672.184937, total loss: 0.509236\n",
      "0.87\n",
      "step 8200, entropy loss: 0.369045, l2_loss: 672.779297, total loss: 0.395956\n",
      "0.94\n",
      "step 8300, entropy loss: 0.567121, l2_loss: 673.420593, total loss: 0.594058\n",
      "0.81\n",
      "step 8400, entropy loss: 0.372148, l2_loss: 674.025696, total loss: 0.399109\n",
      "0.93\n",
      "step 8500, entropy loss: 0.653864, l2_loss: 674.584412, total loss: 0.680847\n",
      "0.81\n",
      "step 8600, entropy loss: 0.570314, l2_loss: 675.133179, total loss: 0.597320\n",
      "0.89\n",
      "step 8700, entropy loss: 0.451447, l2_loss: 675.710876, total loss: 0.478475\n",
      "0.9\n",
      "step 8800, entropy loss: 0.463668, l2_loss: 676.259216, total loss: 0.490719\n",
      "0.82\n",
      "step 8900, entropy loss: 0.477809, l2_loss: 676.800903, total loss: 0.504881\n",
      "0.83\n",
      "step 9000, entropy loss: 0.528644, l2_loss: 677.308228, total loss: 0.555736\n",
      "0.89\n",
      "0.84\n",
      "step 9100, entropy loss: 0.343213, l2_loss: 677.804749, total loss: 0.370325\n",
      "0.86\n",
      "step 9200, entropy loss: 0.390037, l2_loss: 678.304932, total loss: 0.417169\n",
      "0.88\n",
      "step 9300, entropy loss: 0.572304, l2_loss: 678.814148, total loss: 0.599457\n",
      "0.87\n",
      "step 9400, entropy loss: 0.505278, l2_loss: 679.296509, total loss: 0.532450\n",
      "0.85\n",
      "step 9500, entropy loss: 0.381939, l2_loss: 679.745605, total loss: 0.409129\n",
      "0.96\n",
      "step 9600, entropy loss: 0.414342, l2_loss: 680.226501, total loss: 0.441551\n",
      "0.88\n",
      "step 9700, entropy loss: 0.416956, l2_loss: 680.678223, total loss: 0.444183\n",
      "0.9\n",
      "step 9800, entropy loss: 0.522505, l2_loss: 681.143066, total loss: 0.549750\n",
      "0.86\n",
      "step 9900, entropy loss: 0.536376, l2_loss: 681.564148, total loss: 0.563639\n",
      "0.87\n",
      "step 10000, entropy loss: 0.375577, l2_loss: 681.992554, total loss: 0.402857\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.87\n",
      "0.8941\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 4e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 5.Dropout从0.5-->0.6"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 7,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.300455, l2_loss: 721.189392, total loss: 2.350938\n",
      "0.09\n",
      "step 200, entropy loss: 2.298670, l2_loss: 721.095215, total loss: 2.349146\n",
      "0.08\n",
      "step 300, entropy loss: 2.297497, l2_loss: 721.006470, total loss: 2.347967\n",
      "0.11\n",
      "step 400, entropy loss: 2.297602, l2_loss: 720.923035, total loss: 2.348067\n",
      "0.19\n",
      "step 500, entropy loss: 2.294771, l2_loss: 720.843811, total loss: 2.345230\n",
      "0.23\n",
      "step 600, entropy loss: 2.287361, l2_loss: 720.769897, total loss: 2.337815\n",
      "0.27\n",
      "step 700, entropy loss: 2.293079, l2_loss: 720.706543, total loss: 2.343528\n",
      "0.2\n",
      "step 800, entropy loss: 2.273944, l2_loss: 720.649780, total loss: 2.324390\n",
      "0.25\n",
      "step 900, entropy loss: 2.277664, l2_loss: 720.602661, total loss: 2.328107\n",
      "0.3\n",
      "step 1000, entropy loss: 2.278743, l2_loss: 720.564819, total loss: 2.329182\n",
      "0.17\n",
      "0.1841\n",
      "step 1100, entropy loss: 2.278155, l2_loss: 720.532104, total loss: 2.328592\n",
      "0.14\n",
      "step 1200, entropy loss: 2.264668, l2_loss: 720.508972, total loss: 2.315103\n",
      "0.1\n",
      "step 1300, entropy loss: 2.266391, l2_loss: 720.498047, total loss: 2.316826\n",
      "0.21\n",
      "step 1400, entropy loss: 2.256188, l2_loss: 720.497375, total loss: 2.306623\n",
      "0.18\n",
      "step 1500, entropy loss: 2.249356, l2_loss: 720.498535, total loss: 2.299791\n",
      "0.18\n",
      "step 1600, entropy loss: 2.229388, l2_loss: 720.509949, total loss: 2.279824\n",
      "0.25\n",
      "step 1700, entropy loss: 2.251478, l2_loss: 720.524963, total loss: 2.301915\n",
      "0.2\n",
      "step 1800, entropy loss: 2.220253, l2_loss: 720.570984, total loss: 2.270693\n",
      "0.22\n",
      "step 1900, entropy loss: 2.235779, l2_loss: 720.618530, total loss: 2.286222\n",
      "0.28\n",
      "step 2000, entropy loss: 2.236209, l2_loss: 720.666992, total loss: 2.286656\n",
      "0.21\n",
      "0.2225\n",
      "step 2100, entropy loss: 2.192117, l2_loss: 720.743530, total loss: 2.242569\n",
      "0.23\n",
      "step 2200, entropy loss: 2.198832, l2_loss: 720.843323, total loss: 2.249291\n",
      "0.22\n",
      "step 2300, entropy loss: 2.136945, l2_loss: 720.960632, total loss: 2.187413\n",
      "0.24\n",
      "step 2400, entropy loss: 2.146897, l2_loss: 721.082336, total loss: 2.197372\n",
      "0.26\n",
      "step 2500, entropy loss: 2.178939, l2_loss: 721.256165, total loss: 2.229427\n",
      "0.24\n",
      "step 2600, entropy loss: 2.165038, l2_loss: 721.442139, total loss: 2.215539\n",
      "0.2\n",
      "step 2700, entropy loss: 2.152734, l2_loss: 721.630859, total loss: 2.203249\n",
      "0.28\n",
      "step 2800, entropy loss: 2.130803, l2_loss: 721.857544, total loss: 2.181333\n",
      "0.25\n",
      "step 2900, entropy loss: 2.121973, l2_loss: 722.119690, total loss: 2.172522\n",
      "0.27\n",
      "step 3000, entropy loss: 2.062506, l2_loss: 722.407227, total loss: 2.113075\n",
      "0.31\n",
      "0.2616\n",
      "step 3100, entropy loss: 2.078677, l2_loss: 722.746216, total loss: 2.129269\n",
      "0.23\n",
      "step 3200, entropy loss: 2.008248, l2_loss: 723.108582, total loss: 2.058866\n",
      "0.3\n",
      "step 3300, entropy loss: 2.040944, l2_loss: 723.524353, total loss: 2.091591\n",
      "0.26\n",
      "step 3400, entropy loss: 2.091491, l2_loss: 723.998169, total loss: 2.142171\n",
      "0.24\n",
      "step 3500, entropy loss: 1.908223, l2_loss: 724.518127, total loss: 1.958940\n",
      "0.39\n",
      "step 3600, entropy loss: 1.887886, l2_loss: 725.064941, total loss: 1.938641\n",
      "0.31\n",
      "step 3700, entropy loss: 1.840889, l2_loss: 725.680237, total loss: 1.891686\n",
      "0.34\n",
      "step 3800, entropy loss: 1.769288, l2_loss: 726.340515, total loss: 1.820132\n",
      "0.35\n",
      "step 3900, entropy loss: 1.680939, l2_loss: 727.032043, total loss: 1.731832\n",
      "0.43\n",
      "step 4000, entropy loss: 1.752589, l2_loss: 727.778687, total loss: 1.803534\n",
      "0.4\n",
      "0.3528\n",
      "step 4100, entropy loss: 1.783326, l2_loss: 728.502136, total loss: 1.834321\n",
      "0.27\n",
      "step 4200, entropy loss: 1.739205, l2_loss: 729.175537, total loss: 1.790247\n",
      "0.39\n",
      "step 4300, entropy loss: 1.558334, l2_loss: 729.847473, total loss: 1.609423\n",
      "0.45\n",
      "step 4400, entropy loss: 1.611578, l2_loss: 730.528076, total loss: 1.662714\n",
      "0.3\n",
      "step 4500, entropy loss: 1.532412, l2_loss: 731.173706, total loss: 1.583595\n",
      "0.41\n",
      "step 4600, entropy loss: 1.540152, l2_loss: 731.797119, total loss: 1.591378\n",
      "0.4\n",
      "step 4700, entropy loss: 1.607525, l2_loss: 732.405518, total loss: 1.658794\n",
      "0.4\n",
      "step 4800, entropy loss: 1.655379, l2_loss: 733.038391, total loss: 1.706692\n",
      "0.4\n",
      "step 4900, entropy loss: 1.429404, l2_loss: 733.697815, total loss: 1.480763\n",
      "0.49\n",
      "step 5000, entropy loss: 1.392421, l2_loss: 734.407349, total loss: 1.443830\n",
      "0.46\n",
      "0.492\n",
      "step 5100, entropy loss: 1.456583, l2_loss: 735.163086, total loss: 1.508045\n",
      "0.54\n",
      "step 5200, entropy loss: 1.267262, l2_loss: 735.921753, total loss: 1.318777\n",
      "0.53\n",
      "step 5300, entropy loss: 1.124665, l2_loss: 736.740662, total loss: 1.176237\n",
      "0.64\n",
      "step 5400, entropy loss: 1.307668, l2_loss: 737.645264, total loss: 1.359303\n",
      "0.55\n",
      "step 5500, entropy loss: 1.288075, l2_loss: 738.498230, total loss: 1.339770\n",
      "0.63\n",
      "step 5600, entropy loss: 1.145582, l2_loss: 739.439270, total loss: 1.197343\n",
      "0.59\n",
      "step 5700, entropy loss: 1.225125, l2_loss: 740.437866, total loss: 1.276956\n",
      "0.52\n",
      "step 5800, entropy loss: 1.068935, l2_loss: 741.382629, total loss: 1.120832\n",
      "0.64\n",
      "step 5900, entropy loss: 1.032755, l2_loss: 742.385254, total loss: 1.084723\n",
      "0.67\n",
      "step 6000, entropy loss: 1.002959, l2_loss: 743.422607, total loss: 1.054999\n",
      "0.68\n",
      "0.6738\n",
      "step 6100, entropy loss: 0.976130, l2_loss: 744.425354, total loss: 1.028240\n",
      "0.7\n",
      "step 6200, entropy loss: 0.881682, l2_loss: 745.374878, total loss: 0.933858\n",
      "0.68\n",
      "step 6300, entropy loss: 1.066147, l2_loss: 746.360718, total loss: 1.118392\n",
      "0.74\n",
      "step 6400, entropy loss: 0.993318, l2_loss: 747.302246, total loss: 1.045630\n",
      "0.67\n",
      "step 6500, entropy loss: 0.843921, l2_loss: 748.217468, total loss: 0.896296\n",
      "0.75\n",
      "step 6600, entropy loss: 0.775586, l2_loss: 749.159363, total loss: 0.828027\n",
      "0.73\n",
      "step 6700, entropy loss: 0.713584, l2_loss: 750.086792, total loss: 0.766090\n",
      "0.79\n",
      "step 6800, entropy loss: 0.719741, l2_loss: 750.957031, total loss: 0.772308\n",
      "0.8\n",
      "step 6900, entropy loss: 0.929974, l2_loss: 751.848877, total loss: 0.982604\n",
      "0.69\n",
      "step 7000, entropy loss: 0.738877, l2_loss: 752.738892, total loss: 0.791568\n",
      "0.75\n",
      "0.7794\n",
      "step 7100, entropy loss: 0.954002, l2_loss: 753.482422, total loss: 1.006746\n",
      "0.77\n",
      "step 7200, entropy loss: 0.794431, l2_loss: 754.282715, total loss: 0.847230\n",
      "0.86\n",
      "step 7300, entropy loss: 0.455131, l2_loss: 755.088501, total loss: 0.507987\n",
      "0.91\n",
      "step 7400, entropy loss: 0.788915, l2_loss: 755.779785, total loss: 0.841820\n",
      "0.78\n",
      "step 7500, entropy loss: 0.759568, l2_loss: 756.539185, total loss: 0.812525\n",
      "0.78\n",
      "step 7600, entropy loss: 0.530250, l2_loss: 757.256592, total loss: 0.583258\n",
      "0.85\n",
      "step 7700, entropy loss: 0.870644, l2_loss: 757.961426, total loss: 0.923701\n",
      "0.77\n",
      "step 7800, entropy loss: 0.499080, l2_loss: 758.633850, total loss: 0.552185\n",
      "0.87\n",
      "step 7900, entropy loss: 0.615872, l2_loss: 759.332397, total loss: 0.669025\n",
      "0.8\n",
      "step 8000, entropy loss: 0.773723, l2_loss: 759.970764, total loss: 0.826921\n",
      "0.73\n",
      "0.8425\n",
      "step 8100, entropy loss: 0.702779, l2_loss: 760.607605, total loss: 0.756022\n",
      "0.77\n",
      "step 8200, entropy loss: 0.564353, l2_loss: 761.257080, total loss: 0.617641\n",
      "0.84\n",
      "step 8300, entropy loss: 0.765816, l2_loss: 761.862427, total loss: 0.819146\n",
      "0.8\n",
      "step 8400, entropy loss: 0.485883, l2_loss: 762.485779, total loss: 0.539257\n",
      "0.88\n",
      "step 8500, entropy loss: 0.501602, l2_loss: 763.037231, total loss: 0.555014\n",
      "0.89\n",
      "step 8600, entropy loss: 0.438416, l2_loss: 763.634888, total loss: 0.491870\n",
      "0.89\n",
      "step 8700, entropy loss: 0.535278, l2_loss: 764.175049, total loss: 0.588771\n",
      "0.82\n",
      "step 8800, entropy loss: 0.583949, l2_loss: 764.705444, total loss: 0.637479\n",
      "0.84\n",
      "step 8900, entropy loss: 0.407350, l2_loss: 765.232117, total loss: 0.460916\n",
      "0.84\n",
      "step 9000, entropy loss: 0.444492, l2_loss: 765.750854, total loss: 0.498094\n",
      "0.83\n",
      "0.8624\n",
      "step 9100, entropy loss: 0.384565, l2_loss: 766.266785, total loss: 0.438204\n",
      "0.88\n",
      "step 9200, entropy loss: 0.463148, l2_loss: 766.770630, total loss: 0.516822\n",
      "0.83\n",
      "step 9300, entropy loss: 0.326348, l2_loss: 767.233643, total loss: 0.380055\n",
      "0.9\n",
      "step 9400, entropy loss: 0.590911, l2_loss: 767.730042, total loss: 0.644652\n",
      "0.83\n",
      "step 9500, entropy loss: 0.467104, l2_loss: 768.171631, total loss: 0.520876\n",
      "0.82\n",
      "step 9600, entropy loss: 0.359832, l2_loss: 768.652466, total loss: 0.413637\n",
      "0.92\n",
      "step 9700, entropy loss: 0.519342, l2_loss: 769.111267, total loss: 0.573180\n",
      "0.86\n",
      "step 9800, entropy loss: 0.362155, l2_loss: 769.531311, total loss: 0.416023\n",
      "0.91\n",
      "step 9900, entropy loss: 0.490244, l2_loss: 770.005249, total loss: 0.544145\n",
      "0.83\n",
      "step 10000, entropy loss: 0.379403, l2_loss: 770.347290, total loss: 0.433327\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.93\n",
      "0.8874\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(10000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.6})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 6.迭代次数从10000--> 30000"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 8,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.297981, l2_loss: 815.582336, total loss: 2.355071\n",
      "0.16\n",
      "step 200, entropy loss: 2.302473, l2_loss: 815.469055, total loss: 2.359556\n",
      "0.12\n",
      "step 300, entropy loss: 2.299580, l2_loss: 815.357544, total loss: 2.356655\n",
      "0.11\n",
      "step 400, entropy loss: 2.301643, l2_loss: 815.249023, total loss: 2.358710\n",
      "0.12\n",
      "step 500, entropy loss: 2.297080, l2_loss: 815.141846, total loss: 2.354140\n",
      "0.15\n",
      "step 600, entropy loss: 2.303482, l2_loss: 815.036682, total loss: 2.360535\n",
      "0.09\n",
      "step 700, entropy loss: 2.295619, l2_loss: 814.933533, total loss: 2.352664\n",
      "0.15\n",
      "step 800, entropy loss: 2.302337, l2_loss: 814.835754, total loss: 2.359376\n",
      "0.08\n",
      "step 900, entropy loss: 2.297746, l2_loss: 814.740295, total loss: 2.354778\n",
      "0.13\n",
      "step 1000, entropy loss: 2.295234, l2_loss: 814.646301, total loss: 2.352259\n",
      "0.17\n",
      "0.1538\n",
      "step 1100, entropy loss: 2.292372, l2_loss: 814.558289, total loss: 2.349391\n",
      "0.23\n",
      "step 1200, entropy loss: 2.284664, l2_loss: 814.474365, total loss: 2.341677\n",
      "0.23\n",
      "step 1300, entropy loss: 2.289838, l2_loss: 814.397949, total loss: 2.346846\n",
      "0.22\n",
      "step 1400, entropy loss: 2.295460, l2_loss: 814.329163, total loss: 2.352463\n",
      "0.14\n",
      "step 1500, entropy loss: 2.287465, l2_loss: 814.272339, total loss: 2.344464\n",
      "0.22\n",
      "step 1600, entropy loss: 2.274807, l2_loss: 814.216736, total loss: 2.331803\n",
      "0.14\n",
      "step 1700, entropy loss: 2.270705, l2_loss: 814.175598, total loss: 2.327697\n",
      "0.22\n",
      "step 1800, entropy loss: 2.266642, l2_loss: 814.148438, total loss: 2.323632\n",
      "0.22\n",
      "step 1900, entropy loss: 2.275138, l2_loss: 814.133057, total loss: 2.332128\n",
      "0.16\n",
      "step 2000, entropy loss: 2.251271, l2_loss: 814.117065, total loss: 2.308259\n",
      "0.25\n",
      "0.1978\n",
      "step 2100, entropy loss: 2.253346, l2_loss: 814.106934, total loss: 2.310333\n",
      "0.24\n",
      "step 2200, entropy loss: 2.217686, l2_loss: 814.132629, total loss: 2.274676\n",
      "0.21\n",
      "step 2300, entropy loss: 2.216626, l2_loss: 814.155212, total loss: 2.273617\n",
      "0.27\n",
      "step 2400, entropy loss: 2.224441, l2_loss: 814.191956, total loss: 2.281434\n",
      "0.2\n",
      "step 2500, entropy loss: 2.214966, l2_loss: 814.251343, total loss: 2.271964\n",
      "0.2\n",
      "step 2600, entropy loss: 2.208087, l2_loss: 814.333252, total loss: 2.265090\n",
      "0.23\n",
      "step 2700, entropy loss: 2.174259, l2_loss: 814.417908, total loss: 2.231269\n",
      "0.19\n",
      "step 2800, entropy loss: 2.192957, l2_loss: 814.543213, total loss: 2.249975\n",
      "0.16\n",
      "step 2900, entropy loss: 2.141658, l2_loss: 814.703796, total loss: 2.198687\n",
      "0.31\n",
      "step 3000, entropy loss: 2.150153, l2_loss: 814.883301, total loss: 2.207195\n",
      "0.24\n",
      "0.2308\n",
      "step 3100, entropy loss: 2.146484, l2_loss: 815.095764, total loss: 2.203541\n",
      "0.15\n",
      "step 3200, entropy loss: 2.184085, l2_loss: 815.360291, total loss: 2.241160\n",
      "0.15\n",
      "step 3300, entropy loss: 2.091367, l2_loss: 815.628967, total loss: 2.148461\n",
      "0.31\n",
      "step 3400, entropy loss: 2.103469, l2_loss: 815.944031, total loss: 2.160585\n",
      "0.2\n",
      "step 3500, entropy loss: 2.163078, l2_loss: 816.260193, total loss: 2.220216\n",
      "0.23\n",
      "step 3600, entropy loss: 2.075638, l2_loss: 816.581909, total loss: 2.132799\n",
      "0.21\n",
      "step 3700, entropy loss: 2.062937, l2_loss: 816.934448, total loss: 2.120122\n",
      "0.3\n",
      "step 3800, entropy loss: 2.032515, l2_loss: 817.257324, total loss: 2.089723\n",
      "0.24\n",
      "step 3900, entropy loss: 2.114254, l2_loss: 817.602661, total loss: 2.171486\n",
      "0.2\n",
      "step 4000, entropy loss: 1.770468, l2_loss: 817.963501, total loss: 1.827726\n",
      "0.43\n",
      "0.281\n",
      "step 4100, entropy loss: 1.889478, l2_loss: 818.351562, total loss: 1.946763\n",
      "0.31\n",
      "step 4200, entropy loss: 1.804783, l2_loss: 818.815735, total loss: 1.862100\n",
      "0.38\n",
      "step 4300, entropy loss: 1.770729, l2_loss: 819.294434, total loss: 1.828080\n",
      "0.3\n",
      "step 4400, entropy loss: 1.743283, l2_loss: 819.822815, total loss: 1.800671\n",
      "0.36\n",
      "step 4500, entropy loss: 1.834685, l2_loss: 820.458008, total loss: 1.892117\n",
      "0.35\n",
      "step 4600, entropy loss: 1.720441, l2_loss: 821.112122, total loss: 1.777919\n",
      "0.38\n",
      "step 4700, entropy loss: 1.586958, l2_loss: 821.843018, total loss: 1.644487\n",
      "0.41\n",
      "step 4800, entropy loss: 1.596904, l2_loss: 822.581421, total loss: 1.654484\n",
      "0.4\n",
      "step 4900, entropy loss: 1.719768, l2_loss: 823.337708, total loss: 1.777402\n",
      "0.37\n",
      "step 5000, entropy loss: 1.658723, l2_loss: 824.074463, total loss: 1.716409\n",
      "0.34\n",
      "0.4229\n",
      "step 5100, entropy loss: 1.586490, l2_loss: 824.816284, total loss: 1.644227\n",
      "0.46\n",
      "step 5200, entropy loss: 1.452305, l2_loss: 825.571289, total loss: 1.510095\n",
      "0.48\n",
      "step 5300, entropy loss: 1.512646, l2_loss: 826.314819, total loss: 1.570488\n",
      "0.52\n",
      "step 5400, entropy loss: 1.327348, l2_loss: 827.117615, total loss: 1.385246\n",
      "0.51\n",
      "step 5500, entropy loss: 1.246819, l2_loss: 827.889282, total loss: 1.304771\n",
      "0.58\n",
      "step 5600, entropy loss: 1.330735, l2_loss: 828.711304, total loss: 1.388745\n",
      "0.59\n",
      "step 5700, entropy loss: 1.326852, l2_loss: 829.581421, total loss: 1.384923\n",
      "0.56\n",
      "step 5800, entropy loss: 1.378693, l2_loss: 830.493408, total loss: 1.436828\n",
      "0.47\n",
      "step 5900, entropy loss: 1.183055, l2_loss: 831.494568, total loss: 1.241259\n",
      "0.67\n",
      "step 6000, entropy loss: 1.028620, l2_loss: 832.508850, total loss: 1.086895\n",
      "0.75\n",
      "0.6495\n",
      "step 6100, entropy loss: 0.995955, l2_loss: 833.570374, total loss: 1.054305\n",
      "0.75\n",
      "step 6200, entropy loss: 1.035125, l2_loss: 834.632690, total loss: 1.093549\n",
      "0.65\n",
      "step 6300, entropy loss: 1.064063, l2_loss: 835.744324, total loss: 1.122565\n",
      "0.65\n",
      "step 6400, entropy loss: 0.987199, l2_loss: 836.865662, total loss: 1.045779\n",
      "0.72\n",
      "step 6500, entropy loss: 0.883747, l2_loss: 837.952759, total loss: 0.942403\n",
      "0.76\n",
      "step 6600, entropy loss: 0.979164, l2_loss: 838.983093, total loss: 1.037893\n",
      "0.69\n",
      "step 6700, entropy loss: 0.923266, l2_loss: 839.988281, total loss: 0.982065\n",
      "0.72\n",
      "step 6800, entropy loss: 0.809154, l2_loss: 840.958191, total loss: 0.868021\n",
      "0.81\n",
      "step 6900, entropy loss: 0.730721, l2_loss: 841.905334, total loss: 0.789655\n",
      "0.78\n",
      "step 7000, entropy loss: 0.733003, l2_loss: 842.861145, total loss: 0.792004\n",
      "0.79\n",
      "0.7601\n",
      "step 7100, entropy loss: 0.762332, l2_loss: 843.711060, total loss: 0.821392\n",
      "0.81\n",
      "step 7200, entropy loss: 0.727593, l2_loss: 844.565308, total loss: 0.786713\n",
      "0.77\n",
      "step 7300, entropy loss: 0.723063, l2_loss: 845.386719, total loss: 0.782240\n",
      "0.81\n",
      "step 7400, entropy loss: 0.740240, l2_loss: 846.164062, total loss: 0.799471\n",
      "0.76\n",
      "step 7500, entropy loss: 0.795797, l2_loss: 846.896667, total loss: 0.855079\n",
      "0.71\n",
      "step 7600, entropy loss: 0.808121, l2_loss: 847.675415, total loss: 0.867458\n",
      "0.77\n",
      "step 7700, entropy loss: 0.528380, l2_loss: 848.458313, total loss: 0.587772\n",
      "0.81\n",
      "step 7800, entropy loss: 0.587257, l2_loss: 849.177368, total loss: 0.646699\n",
      "0.87\n",
      "step 7900, entropy loss: 0.683401, l2_loss: 849.849548, total loss: 0.742890\n",
      "0.88\n",
      "step 8000, entropy loss: 0.555249, l2_loss: 850.535706, total loss: 0.614786\n",
      "0.82\n",
      "0.8434\n",
      "step 8100, entropy loss: 0.662778, l2_loss: 851.174866, total loss: 0.722360\n",
      "0.81\n",
      "step 8200, entropy loss: 0.660604, l2_loss: 851.822205, total loss: 0.720232\n",
      "0.8\n",
      "step 8300, entropy loss: 0.394814, l2_loss: 852.406494, total loss: 0.454482\n",
      "0.88\n",
      "step 8400, entropy loss: 0.437020, l2_loss: 853.025940, total loss: 0.496732\n",
      "0.88\n",
      "step 8500, entropy loss: 0.485731, l2_loss: 853.652832, total loss: 0.545487\n",
      "0.86\n",
      "step 8600, entropy loss: 0.398964, l2_loss: 854.202148, total loss: 0.458758\n",
      "0.92\n",
      "step 8700, entropy loss: 0.534365, l2_loss: 854.762573, total loss: 0.594198\n",
      "0.85\n",
      "step 8800, entropy loss: 0.356273, l2_loss: 855.295776, total loss: 0.416144\n",
      "0.93\n",
      "step 8900, entropy loss: 0.422481, l2_loss: 855.855896, total loss: 0.482391\n",
      "0.9\n",
      "step 9000, entropy loss: 0.377313, l2_loss: 856.384277, total loss: 0.437260\n",
      "0.88\n",
      "0.8719\n",
      "step 9100, entropy loss: 0.523980, l2_loss: 856.895020, total loss: 0.583962\n",
      "0.81\n",
      "step 9200, entropy loss: 0.488257, l2_loss: 857.383240, total loss: 0.548274\n",
      "0.88\n",
      "step 9300, entropy loss: 0.345024, l2_loss: 857.868652, total loss: 0.405075\n",
      "0.9\n",
      "step 9400, entropy loss: 0.462668, l2_loss: 858.344849, total loss: 0.522752\n",
      "0.85\n",
      "step 9500, entropy loss: 0.419650, l2_loss: 858.849243, total loss: 0.479770\n",
      "0.94\n",
      "step 9600, entropy loss: 0.440360, l2_loss: 859.316589, total loss: 0.500512\n",
      "0.9\n",
      "step 9700, entropy loss: 0.356128, l2_loss: 859.717407, total loss: 0.416308\n",
      "0.9\n",
      "step 9800, entropy loss: 0.416103, l2_loss: 860.150208, total loss: 0.476314\n",
      "0.89\n",
      "step 9900, entropy loss: 0.405488, l2_loss: 860.586914, total loss: 0.465729\n",
      "0.86\n",
      "step 10000, entropy loss: 0.309264, l2_loss: 860.974487, total loss: 0.369532\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.91\n",
      "0.8923\n",
      "step 10100, entropy loss: 0.488102, l2_loss: 861.426453, total loss: 0.548402\n",
      "0.83\n",
      "step 10200, entropy loss: 0.299561, l2_loss: 861.798096, total loss: 0.359887\n",
      "0.94\n",
      "step 10300, entropy loss: 0.270800, l2_loss: 862.225220, total loss: 0.331156\n",
      "0.89\n",
      "step 10400, entropy loss: 0.495505, l2_loss: 862.596313, total loss: 0.555887\n",
      "0.86\n",
      "step 10500, entropy loss: 0.379872, l2_loss: 862.976135, total loss: 0.440280\n",
      "0.89\n",
      "step 10600, entropy loss: 0.462765, l2_loss: 863.368835, total loss: 0.523201\n",
      "0.91\n",
      "step 10700, entropy loss: 0.431561, l2_loss: 863.727234, total loss: 0.492022\n",
      "0.91\n",
      "step 10800, entropy loss: 0.400901, l2_loss: 864.070007, total loss: 0.461386\n",
      "0.9\n",
      "step 10900, entropy loss: 0.268109, l2_loss: 864.420288, total loss: 0.328618\n",
      "0.92\n",
      "step 11000, entropy loss: 0.505598, l2_loss: 864.757812, total loss: 0.566131\n",
      "0.89\n",
      "0.8954\n",
      "step 11100, entropy loss: 0.282996, l2_loss: 865.089966, total loss: 0.343552\n",
      "0.95\n",
      "step 11200, entropy loss: 0.275467, l2_loss: 865.409851, total loss: 0.336046\n",
      "0.94\n",
      "step 11300, entropy loss: 0.541494, l2_loss: 865.741089, total loss: 0.602095\n",
      "0.87\n",
      "step 11400, entropy loss: 0.235553, l2_loss: 866.062134, total loss: 0.296177\n",
      "0.95\n",
      "step 11500, entropy loss: 0.245576, l2_loss: 866.379211, total loss: 0.306223\n",
      "0.95\n",
      "step 11600, entropy loss: 0.536348, l2_loss: 866.707520, total loss: 0.597017\n",
      "0.86\n",
      "step 11700, entropy loss: 0.248744, l2_loss: 867.002808, total loss: 0.309434\n",
      "0.94\n",
      "step 11800, entropy loss: 0.286589, l2_loss: 867.279114, total loss: 0.347298\n",
      "0.94\n",
      "step 11900, entropy loss: 0.345903, l2_loss: 867.594055, total loss: 0.406635\n",
      "0.89\n",
      "step 12000, entropy loss: 0.337726, l2_loss: 867.878418, total loss: 0.398477\n",
      "0.91\n",
      "0.917\n",
      "step 12100, entropy loss: 0.231667, l2_loss: 868.160645, total loss: 0.292439\n",
      "0.94\n",
      "step 12200, entropy loss: 0.248360, l2_loss: 868.463379, total loss: 0.309153\n",
      "0.93\n",
      "step 12300, entropy loss: 0.374035, l2_loss: 868.709900, total loss: 0.434845\n",
      "0.92\n",
      "step 12400, entropy loss: 0.311072, l2_loss: 868.995239, total loss: 0.371902\n",
      "0.91\n",
      "step 12500, entropy loss: 0.275959, l2_loss: 869.262939, total loss: 0.336807\n",
      "0.94\n",
      "step 12600, entropy loss: 0.354002, l2_loss: 869.508789, total loss: 0.414868\n",
      "0.9\n",
      "step 12700, entropy loss: 0.335667, l2_loss: 869.782959, total loss: 0.396552\n",
      "0.88\n",
      "step 12800, entropy loss: 0.181201, l2_loss: 870.038208, total loss: 0.242104\n",
      "0.97\n",
      "step 12900, entropy loss: 0.391269, l2_loss: 870.313538, total loss: 0.452191\n",
      "0.89\n",
      "step 13000, entropy loss: 0.262670, l2_loss: 870.539673, total loss: 0.323608\n",
      "0.96\n",
      "0.9191\n",
      "step 13100, entropy loss: 0.246504, l2_loss: 870.776062, total loss: 0.307458\n",
      "0.94\n",
      "step 13200, entropy loss: 0.250921, l2_loss: 870.999878, total loss: 0.311891\n",
      "0.95\n",
      "step 13300, entropy loss: 0.364246, l2_loss: 871.259949, total loss: 0.425234\n",
      "0.96\n",
      "step 13400, entropy loss: 0.246954, l2_loss: 871.476440, total loss: 0.307957\n",
      "0.93\n",
      "step 13500, entropy loss: 0.313158, l2_loss: 871.707153, total loss: 0.374178\n",
      "0.91\n",
      "step 13600, entropy loss: 0.274308, l2_loss: 871.925842, total loss: 0.335343\n",
      "0.94\n",
      "step 13700, entropy loss: 0.392826, l2_loss: 872.159546, total loss: 0.453877\n",
      "0.88\n",
      "step 13800, entropy loss: 0.392042, l2_loss: 872.351440, total loss: 0.453106\n",
      "0.91\n",
      "step 13900, entropy loss: 0.369801, l2_loss: 872.579529, total loss: 0.430881\n",
      "0.91\n",
      "step 14000, entropy loss: 0.201918, l2_loss: 872.820129, total loss: 0.263016\n",
      "0.95\n",
      "0.9222\n",
      "step 14100, entropy loss: 0.273649, l2_loss: 873.057190, total loss: 0.334763\n",
      "0.91\n",
      "step 14200, entropy loss: 0.184900, l2_loss: 873.241455, total loss: 0.246027\n",
      "0.96\n",
      "step 14300, entropy loss: 0.397224, l2_loss: 873.456726, total loss: 0.458366\n",
      "0.9\n",
      "step 14400, entropy loss: 0.182927, l2_loss: 873.659180, total loss: 0.244083\n",
      "0.97\n",
      "step 14500, entropy loss: 0.281726, l2_loss: 873.881042, total loss: 0.342898\n",
      "0.98\n",
      "step 14600, entropy loss: 0.186277, l2_loss: 874.063965, total loss: 0.247462\n",
      "0.93\n",
      "step 14700, entropy loss: 0.445027, l2_loss: 874.299622, total loss: 0.506228\n",
      "0.87\n",
      "step 14800, entropy loss: 0.504867, l2_loss: 874.490234, total loss: 0.566081\n",
      "0.86\n",
      "step 14900, entropy loss: 0.293379, l2_loss: 874.693848, total loss: 0.354608\n",
      "0.94\n",
      "step 15000, entropy loss: 0.349948, l2_loss: 874.859619, total loss: 0.411188\n",
      "0.9\n",
      "0.9265\n",
      "step 15100, entropy loss: 0.348683, l2_loss: 875.050537, total loss: 0.409937\n",
      "0.87\n",
      "step 15200, entropy loss: 0.287354, l2_loss: 875.225952, total loss: 0.348620\n",
      "0.95\n",
      "step 15300, entropy loss: 0.201814, l2_loss: 875.455322, total loss: 0.263096\n",
      "0.96\n",
      "step 15400, entropy loss: 0.285230, l2_loss: 875.620972, total loss: 0.346523\n",
      "0.93\n",
      "step 15500, entropy loss: 0.176724, l2_loss: 875.833130, total loss: 0.238032\n",
      "0.97\n",
      "step 15600, entropy loss: 0.209208, l2_loss: 876.029724, total loss: 0.270530\n",
      "0.94\n",
      "step 15700, entropy loss: 0.268010, l2_loss: 876.220703, total loss: 0.329345\n",
      "0.95\n",
      "step 15800, entropy loss: 0.301644, l2_loss: 876.376587, total loss: 0.362990\n",
      "0.93\n",
      "step 15900, entropy loss: 0.279367, l2_loss: 876.533508, total loss: 0.340724\n",
      "0.94\n",
      "step 16000, entropy loss: 0.220307, l2_loss: 876.743774, total loss: 0.281679\n",
      "0.95\n",
      "0.9293\n",
      "step 16100, entropy loss: 0.226012, l2_loss: 876.902466, total loss: 0.287395\n",
      "0.94\n",
      "step 16200, entropy loss: 0.249147, l2_loss: 877.070557, total loss: 0.310542\n",
      "0.95\n",
      "step 16300, entropy loss: 0.149172, l2_loss: 877.258179, total loss: 0.210580\n",
      "0.97\n",
      "step 16400, entropy loss: 0.222941, l2_loss: 877.432068, total loss: 0.284361\n",
      "0.94\n",
      "step 16500, entropy loss: 0.161050, l2_loss: 877.578613, total loss: 0.222481\n",
      "0.99\n",
      "step 16600, entropy loss: 0.198282, l2_loss: 877.734619, total loss: 0.259723\n",
      "0.95\n",
      "step 16700, entropy loss: 0.194921, l2_loss: 877.913696, total loss: 0.256375\n",
      "0.97\n",
      "step 16800, entropy loss: 0.199602, l2_loss: 878.061707, total loss: 0.261066\n",
      "0.95\n",
      "step 16900, entropy loss: 0.087954, l2_loss: 878.253906, total loss: 0.149432\n",
      "0.99\n",
      "step 17000, entropy loss: 0.183377, l2_loss: 878.409058, total loss: 0.244866\n",
      "0.96\n",
      "0.9304\n",
      "step 17100, entropy loss: 0.213934, l2_loss: 878.567749, total loss: 0.275434\n",
      "0.97\n",
      "step 17200, entropy loss: 0.251634, l2_loss: 878.749756, total loss: 0.313146\n",
      "0.92\n",
      "step 17300, entropy loss: 0.254849, l2_loss: 878.921997, total loss: 0.316373\n",
      "0.97\n",
      "step 17400, entropy loss: 0.114208, l2_loss: 879.036621, total loss: 0.175740\n",
      "0.98\n",
      "step 17500, entropy loss: 0.219664, l2_loss: 879.180725, total loss: 0.281207\n",
      "0.94\n",
      "step 17600, entropy loss: 0.121705, l2_loss: 879.342041, total loss: 0.183259\n",
      "0.98\n",
      "step 17700, entropy loss: 0.337499, l2_loss: 879.487427, total loss: 0.399063\n",
      "0.88\n",
      "step 17800, entropy loss: 0.190196, l2_loss: 879.628052, total loss: 0.251770\n",
      "0.93\n",
      "step 17900, entropy loss: 0.250924, l2_loss: 879.781067, total loss: 0.312508\n",
      "0.91\n",
      "step 18000, entropy loss: 0.302987, l2_loss: 879.945923, total loss: 0.364583\n",
      "0.9\n",
      "0.9376\n",
      "step 18100, entropy loss: 0.267047, l2_loss: 880.067383, total loss: 0.328652\n",
      "0.94\n",
      "step 18200, entropy loss: 0.091645, l2_loss: 880.203979, total loss: 0.153259\n",
      "0.98\n",
      "step 18300, entropy loss: 0.221456, l2_loss: 880.330444, total loss: 0.283079\n",
      "0.96\n",
      "step 18400, entropy loss: 0.104792, l2_loss: 880.494080, total loss: 0.166427\n",
      "0.98\n",
      "step 18500, entropy loss: 0.253410, l2_loss: 880.650452, total loss: 0.315055\n",
      "0.95\n",
      "step 18600, entropy loss: 0.198156, l2_loss: 880.779114, total loss: 0.259811\n",
      "0.94\n",
      "step 18700, entropy loss: 0.165200, l2_loss: 880.921387, total loss: 0.226865\n",
      "0.98\n",
      "step 18800, entropy loss: 0.203207, l2_loss: 881.028748, total loss: 0.264879\n",
      "0.95\n",
      "step 18900, entropy loss: 0.191604, l2_loss: 881.157471, total loss: 0.253285\n",
      "0.93\n",
      "step 19000, entropy loss: 0.221020, l2_loss: 881.310913, total loss: 0.282712\n",
      "0.96\n",
      "0.9437\n",
      "step 19100, entropy loss: 0.181047, l2_loss: 881.441040, total loss: 0.242748\n",
      "0.94\n",
      "step 19200, entropy loss: 0.198988, l2_loss: 881.569946, total loss: 0.260698\n",
      "0.93\n",
      "step 19300, entropy loss: 0.143532, l2_loss: 881.692139, total loss: 0.205250\n",
      "0.96\n",
      "step 19400, entropy loss: 0.163373, l2_loss: 881.819397, total loss: 0.225101\n",
      "0.98\n",
      "step 19500, entropy loss: 0.268803, l2_loss: 881.941406, total loss: 0.330539\n",
      "0.92\n",
      "step 19600, entropy loss: 0.206308, l2_loss: 882.081604, total loss: 0.268053\n",
      "0.95\n",
      "step 19700, entropy loss: 0.210543, l2_loss: 882.180603, total loss: 0.272296\n",
      "0.94\n",
      "step 19800, entropy loss: 0.173917, l2_loss: 882.305481, total loss: 0.235678\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.96\n",
      "step 19900, entropy loss: 0.221915, l2_loss: 882.452271, total loss: 0.283686\n",
      "0.92\n",
      "step 20000, entropy loss: 0.171784, l2_loss: 882.558472, total loss: 0.233563\n",
      "0.96\n",
      "0.9456\n",
      "step 20100, entropy loss: 0.212408, l2_loss: 882.660950, total loss: 0.274194\n",
      "0.97\n",
      "step 20200, entropy loss: 0.144867, l2_loss: 882.819519, total loss: 0.206664\n",
      "0.96\n",
      "step 20300, entropy loss: 0.213349, l2_loss: 882.928345, total loss: 0.275154\n",
      "0.96\n",
      "step 20400, entropy loss: 0.170796, l2_loss: 883.019897, total loss: 0.232608\n",
      "0.94\n",
      "step 20500, entropy loss: 0.192190, l2_loss: 883.148376, total loss: 0.254010\n",
      "0.95\n",
      "step 20600, entropy loss: 0.098488, l2_loss: 883.279114, total loss: 0.160318\n",
      "0.99\n",
      "step 20700, entropy loss: 0.267188, l2_loss: 883.355408, total loss: 0.329023\n",
      "0.93\n",
      "step 20800, entropy loss: 0.204217, l2_loss: 883.466187, total loss: 0.266060\n",
      "0.94\n",
      "step 20900, entropy loss: 0.224964, l2_loss: 883.589294, total loss: 0.286816\n",
      "0.98\n",
      "step 21000, entropy loss: 0.182753, l2_loss: 883.716370, total loss: 0.244613\n",
      "0.95\n",
      "0.9429\n",
      "step 21100, entropy loss: 0.270375, l2_loss: 883.822754, total loss: 0.332243\n",
      "0.94\n",
      "step 21200, entropy loss: 0.188151, l2_loss: 883.912354, total loss: 0.250025\n",
      "0.96\n",
      "step 21300, entropy loss: 0.228434, l2_loss: 884.035217, total loss: 0.290317\n",
      "0.94\n",
      "step 21400, entropy loss: 0.256308, l2_loss: 884.137939, total loss: 0.318198\n",
      "0.93\n",
      "step 21500, entropy loss: 0.236469, l2_loss: 884.216919, total loss: 0.298364\n",
      "0.93\n",
      "step 21600, entropy loss: 0.174484, l2_loss: 884.346802, total loss: 0.236388\n",
      "0.93\n",
      "step 21700, entropy loss: 0.165142, l2_loss: 884.456360, total loss: 0.227054\n",
      "0.97\n",
      "step 21800, entropy loss: 0.185883, l2_loss: 884.568848, total loss: 0.247803\n",
      "0.96\n",
      "step 21900, entropy loss: 0.186879, l2_loss: 884.650085, total loss: 0.248804\n",
      "0.94\n",
      "step 22000, entropy loss: 0.119852, l2_loss: 884.742615, total loss: 0.181784\n",
      "0.96\n",
      "0.9485\n",
      "step 22100, entropy loss: 0.252192, l2_loss: 884.832642, total loss: 0.314130\n",
      "0.93\n",
      "step 22200, entropy loss: 0.154352, l2_loss: 884.925781, total loss: 0.216297\n",
      "0.94\n",
      "step 22300, entropy loss: 0.077172, l2_loss: 885.041138, total loss: 0.139125\n",
      "0.98\n",
      "step 22400, entropy loss: 0.305055, l2_loss: 885.133057, total loss: 0.367015\n",
      "0.96\n",
      "step 22500, entropy loss: 0.275704, l2_loss: 885.244629, total loss: 0.337671\n",
      "0.91\n",
      "step 22600, entropy loss: 0.090734, l2_loss: 885.336792, total loss: 0.152707\n",
      "0.98\n",
      "step 22700, entropy loss: 0.258921, l2_loss: 885.425293, total loss: 0.320900\n",
      "0.96\n",
      "step 22800, entropy loss: 0.193444, l2_loss: 885.526489, total loss: 0.255431\n",
      "0.94\n",
      "step 22900, entropy loss: 0.204690, l2_loss: 885.630920, total loss: 0.266684\n",
      "0.95\n",
      "step 23000, entropy loss: 0.143058, l2_loss: 885.684448, total loss: 0.205056\n",
      "0.95\n",
      "0.9521\n",
      "step 23100, entropy loss: 0.115778, l2_loss: 885.777527, total loss: 0.177782\n",
      "0.97\n",
      "step 23200, entropy loss: 0.158468, l2_loss: 885.875122, total loss: 0.220480\n",
      "0.95\n",
      "step 23300, entropy loss: 0.230636, l2_loss: 885.979004, total loss: 0.292654\n",
      "0.93\n",
      "step 23400, entropy loss: 0.135122, l2_loss: 886.084045, total loss: 0.197148\n",
      "0.97\n",
      "step 23500, entropy loss: 0.086426, l2_loss: 886.172058, total loss: 0.148458\n",
      "0.97\n",
      "step 23600, entropy loss: 0.174313, l2_loss: 886.247131, total loss: 0.236351\n",
      "0.96\n",
      "step 23700, entropy loss: 0.169388, l2_loss: 886.347717, total loss: 0.231432\n",
      "0.97\n",
      "step 23800, entropy loss: 0.254265, l2_loss: 886.406433, total loss: 0.316313\n",
      "0.96\n",
      "step 23900, entropy loss: 0.176002, l2_loss: 886.511475, total loss: 0.238058\n",
      "0.96\n",
      "step 24000, entropy loss: 0.224537, l2_loss: 886.608398, total loss: 0.286599\n",
      "0.96\n",
      "0.9536\n",
      "step 24100, entropy loss: 0.148789, l2_loss: 886.682190, total loss: 0.210857\n",
      "0.97\n",
      "step 24200, entropy loss: 0.209265, l2_loss: 886.768555, total loss: 0.271338\n",
      "0.94\n",
      "step 24300, entropy loss: 0.163873, l2_loss: 886.874695, total loss: 0.225954\n",
      "0.96\n",
      "step 24400, entropy loss: 0.150078, l2_loss: 886.950256, total loss: 0.212165\n",
      "0.97\n",
      "step 24500, entropy loss: 0.071390, l2_loss: 887.052063, total loss: 0.133484\n",
      "0.98\n",
      "step 24600, entropy loss: 0.065398, l2_loss: 887.117554, total loss: 0.127496\n",
      "0.99\n",
      "step 24700, entropy loss: 0.427793, l2_loss: 887.183105, total loss: 0.489896\n",
      "0.92\n",
      "step 24800, entropy loss: 0.167002, l2_loss: 887.284668, total loss: 0.229112\n",
      "0.96\n",
      "step 24900, entropy loss: 0.224544, l2_loss: 887.353882, total loss: 0.286659\n",
      "0.95\n",
      "step 25000, entropy loss: 0.175510, l2_loss: 887.451416, total loss: 0.237632\n",
      "0.97\n",
      "0.9518\n",
      "step 25100, entropy loss: 0.209159, l2_loss: 887.532715, total loss: 0.271286\n",
      "0.96\n",
      "step 25200, entropy loss: 0.282148, l2_loss: 887.602051, total loss: 0.344280\n",
      "0.95\n",
      "step 25300, entropy loss: 0.126967, l2_loss: 887.654541, total loss: 0.189103\n",
      "0.96\n",
      "step 25400, entropy loss: 0.230810, l2_loss: 887.723267, total loss: 0.292951\n",
      "0.92\n",
      "step 25500, entropy loss: 0.165667, l2_loss: 887.796143, total loss: 0.227813\n",
      "0.97\n",
      "step 25600, entropy loss: 0.106751, l2_loss: 887.880493, total loss: 0.168903\n",
      "0.96\n",
      "step 25700, entropy loss: 0.156662, l2_loss: 887.942566, total loss: 0.218818\n",
      "0.94\n",
      "step 25800, entropy loss: 0.093655, l2_loss: 888.037170, total loss: 0.155818\n",
      "0.98\n",
      "step 25900, entropy loss: 0.137032, l2_loss: 888.108398, total loss: 0.199199\n",
      "0.98\n",
      "step 26000, entropy loss: 0.082780, l2_loss: 888.171692, total loss: 0.144952\n",
      "0.99\n",
      "0.9546\n",
      "step 26100, entropy loss: 0.206331, l2_loss: 888.262207, total loss: 0.268510\n",
      "0.97\n",
      "step 26200, entropy loss: 0.152592, l2_loss: 888.336609, total loss: 0.214776\n",
      "0.95\n",
      "step 26300, entropy loss: 0.107745, l2_loss: 888.410034, total loss: 0.169934\n",
      "0.97\n",
      "step 26400, entropy loss: 0.142800, l2_loss: 888.485352, total loss: 0.204994\n",
      "0.96\n",
      "step 26500, entropy loss: 0.132642, l2_loss: 888.577759, total loss: 0.194842\n",
      "0.95\n",
      "step 26600, entropy loss: 0.134003, l2_loss: 888.668213, total loss: 0.196210\n",
      "0.99\n",
      "step 26700, entropy loss: 0.235807, l2_loss: 888.727173, total loss: 0.298018\n",
      "0.95\n",
      "step 26800, entropy loss: 0.161810, l2_loss: 888.770813, total loss: 0.224024\n",
      "0.96\n",
      "step 26900, entropy loss: 0.179774, l2_loss: 888.802124, total loss: 0.241990\n",
      "0.99\n",
      "step 27000, entropy loss: 0.116895, l2_loss: 888.888672, total loss: 0.179117\n",
      "0.96\n",
      "0.9581\n",
      "step 27100, entropy loss: 0.119424, l2_loss: 888.956177, total loss: 0.181651\n",
      "0.96\n",
      "step 27200, entropy loss: 0.130364, l2_loss: 889.013916, total loss: 0.192595\n",
      "0.98\n",
      "step 27300, entropy loss: 0.109217, l2_loss: 889.080383, total loss: 0.171452\n",
      "0.98\n",
      "step 27400, entropy loss: 0.197606, l2_loss: 889.180420, total loss: 0.259848\n",
      "0.94\n",
      "step 27500, entropy loss: 0.326410, l2_loss: 889.223633, total loss: 0.388656\n",
      "0.93\n",
      "step 27600, entropy loss: 0.140997, l2_loss: 889.297180, total loss: 0.203247\n",
      "0.98\n",
      "step 27700, entropy loss: 0.181678, l2_loss: 889.355957, total loss: 0.243932\n",
      "0.97\n",
      "step 27800, entropy loss: 0.104920, l2_loss: 889.411072, total loss: 0.167179\n",
      "0.98\n",
      "step 27900, entropy loss: 0.240593, l2_loss: 889.460693, total loss: 0.302855\n",
      "0.96\n",
      "step 28000, entropy loss: 0.050153, l2_loss: 889.546387, total loss: 0.112421\n",
      "1.0\n",
      "0.957\n",
      "step 28100, entropy loss: 0.199267, l2_loss: 889.623901, total loss: 0.261541\n",
      "0.95\n",
      "step 28200, entropy loss: 0.123066, l2_loss: 889.676270, total loss: 0.185344\n",
      "0.96\n",
      "step 28300, entropy loss: 0.150564, l2_loss: 889.730164, total loss: 0.212845\n",
      "0.98\n",
      "step 28400, entropy loss: 0.198860, l2_loss: 889.789490, total loss: 0.261146\n",
      "0.94\n",
      "step 28500, entropy loss: 0.122634, l2_loss: 889.860229, total loss: 0.184924\n",
      "0.96\n",
      "step 28600, entropy loss: 0.117592, l2_loss: 889.922974, total loss: 0.179887\n",
      "0.95\n",
      "step 28700, entropy loss: 0.154000, l2_loss: 889.981567, total loss: 0.216299\n",
      "0.99\n",
      "step 28800, entropy loss: 0.171362, l2_loss: 890.058472, total loss: 0.233666\n",
      "0.96\n",
      "step 28900, entropy loss: 0.167423, l2_loss: 890.101929, total loss: 0.229730\n",
      "0.98\n",
      "step 29000, entropy loss: 0.119424, l2_loss: 890.149170, total loss: 0.181735\n",
      "0.97\n",
      "0.9579\n",
      "step 29100, entropy loss: 0.192425, l2_loss: 890.238037, total loss: 0.254742\n",
      "0.96\n",
      "step 29200, entropy loss: 0.160062, l2_loss: 890.293884, total loss: 0.222383\n",
      "0.94\n",
      "step 29300, entropy loss: 0.157175, l2_loss: 890.353333, total loss: 0.219500\n",
      "0.98\n",
      "step 29400, entropy loss: 0.117532, l2_loss: 890.410889, total loss: 0.179861\n",
      "0.98\n",
      "step 29500, entropy loss: 0.263868, l2_loss: 890.452515, total loss: 0.326199\n",
      "0.95\n",
      "step 29600, entropy loss: 0.164478, l2_loss: 890.514526, total loss: 0.226814\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.96\n",
      "step 29700, entropy loss: 0.160154, l2_loss: 890.598511, total loss: 0.222496\n",
      "0.96\n",
      "step 29800, entropy loss: 0.152268, l2_loss: 890.624390, total loss: 0.214612\n",
      "0.98\n",
      "step 29900, entropy loss: 0.219131, l2_loss: 890.679932, total loss: 0.281478\n",
      "0.95\n",
      "step 30000, entropy loss: 0.182669, l2_loss: 890.735596, total loss: 0.245020\n",
      "0.95\n",
      "0.9607\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(30000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 迭代次数从3w-->6w"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 9,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 100, entropy loss: 2.304106, l2_loss: 906.832092, total loss: 2.367584\n",
      "0.12\n",
      "step 200, entropy loss: 2.304860, l2_loss: 906.707642, total loss: 2.368329\n",
      "0.06\n",
      "step 300, entropy loss: 2.295582, l2_loss: 906.586609, total loss: 2.359043\n",
      "0.1\n",
      "step 400, entropy loss: 2.296325, l2_loss: 906.470215, total loss: 2.359778\n",
      "0.14\n",
      "step 500, entropy loss: 2.301021, l2_loss: 906.357239, total loss: 2.364466\n",
      "0.09\n",
      "step 600, entropy loss: 2.293843, l2_loss: 906.249512, total loss: 2.357281\n",
      "0.22\n",
      "step 700, entropy loss: 2.293152, l2_loss: 906.149292, total loss: 2.356582\n",
      "0.2\n",
      "step 800, entropy loss: 2.298354, l2_loss: 906.053711, total loss: 2.361778\n",
      "0.21\n",
      "step 900, entropy loss: 2.287083, l2_loss: 905.966675, total loss: 2.350501\n",
      "0.15\n",
      "step 1000, entropy loss: 2.281933, l2_loss: 905.885254, total loss: 2.345345\n",
      "0.2\n",
      "0.2071\n",
      "step 1100, entropy loss: 2.292841, l2_loss: 905.814941, total loss: 2.356248\n",
      "0.16\n",
      "step 1200, entropy loss: 2.270530, l2_loss: 905.758301, total loss: 2.333933\n",
      "0.14\n",
      "step 1300, entropy loss: 2.273364, l2_loss: 905.710571, total loss: 2.336764\n",
      "0.18\n",
      "step 1400, entropy loss: 2.251229, l2_loss: 905.673340, total loss: 2.314626\n",
      "0.25\n",
      "step 1500, entropy loss: 2.235669, l2_loss: 905.643799, total loss: 2.299064\n",
      "0.22\n",
      "step 1600, entropy loss: 2.252056, l2_loss: 905.629883, total loss: 2.315450\n",
      "0.2\n",
      "step 1700, entropy loss: 2.260565, l2_loss: 905.611023, total loss: 2.323957\n",
      "0.21\n",
      "step 1800, entropy loss: 2.213494, l2_loss: 905.624390, total loss: 2.276888\n",
      "0.24\n",
      "step 1900, entropy loss: 2.219564, l2_loss: 905.646362, total loss: 2.282959\n",
      "0.17\n",
      "step 2000, entropy loss: 2.228858, l2_loss: 905.683350, total loss: 2.292256\n",
      "0.21\n",
      "0.2116\n",
      "step 2100, entropy loss: 2.239492, l2_loss: 905.744934, total loss: 2.302895\n",
      "0.16\n",
      "step 2200, entropy loss: 2.211466, l2_loss: 905.825562, total loss: 2.274874\n",
      "0.13\n",
      "step 2300, entropy loss: 2.180120, l2_loss: 905.927124, total loss: 2.243535\n",
      "0.21\n",
      "step 2400, entropy loss: 2.200455, l2_loss: 906.054199, total loss: 2.263879\n",
      "0.17\n",
      "step 2500, entropy loss: 2.223986, l2_loss: 906.219116, total loss: 2.287421\n",
      "0.17\n",
      "step 2600, entropy loss: 2.088083, l2_loss: 906.397400, total loss: 2.151531\n",
      "0.28\n",
      "step 2700, entropy loss: 2.127201, l2_loss: 906.619446, total loss: 2.190664\n",
      "0.25\n",
      "step 2800, entropy loss: 2.143849, l2_loss: 906.868713, total loss: 2.207330\n",
      "0.19\n",
      "step 2900, entropy loss: 2.108336, l2_loss: 907.148315, total loss: 2.171837\n",
      "0.23\n",
      "step 3000, entropy loss: 2.052414, l2_loss: 907.451904, total loss: 2.115936\n",
      "0.26\n",
      "0.2398\n",
      "step 3100, entropy loss: 2.099170, l2_loss: 907.759216, total loss: 2.162713\n",
      "0.18\n",
      "step 3200, entropy loss: 1.994676, l2_loss: 908.053284, total loss: 2.058240\n",
      "0.26\n",
      "step 3300, entropy loss: 1.928766, l2_loss: 908.354248, total loss: 1.992350\n",
      "0.28\n",
      "step 3400, entropy loss: 2.072883, l2_loss: 908.604004, total loss: 2.136485\n",
      "0.32\n",
      "step 3500, entropy loss: 2.024098, l2_loss: 908.885864, total loss: 2.087720\n",
      "0.23\n",
      "step 3600, entropy loss: 1.982141, l2_loss: 909.218201, total loss: 2.045787\n",
      "0.28\n",
      "step 3700, entropy loss: 1.967227, l2_loss: 909.534058, total loss: 2.030894\n",
      "0.23\n",
      "step 3800, entropy loss: 1.891541, l2_loss: 909.894592, total loss: 1.955234\n",
      "0.35\n",
      "step 3900, entropy loss: 1.815229, l2_loss: 910.266602, total loss: 1.878948\n",
      "0.31\n",
      "step 4000, entropy loss: 1.848294, l2_loss: 910.708557, total loss: 1.912044\n",
      "0.33\n",
      "0.3307\n",
      "step 4100, entropy loss: 1.796272, l2_loss: 911.201233, total loss: 1.860057\n",
      "0.36\n",
      "step 4200, entropy loss: 1.754206, l2_loss: 911.696533, total loss: 1.818025\n",
      "0.34\n",
      "step 4300, entropy loss: 1.752371, l2_loss: 912.256104, total loss: 1.816229\n",
      "0.33\n",
      "step 4400, entropy loss: 1.768929, l2_loss: 912.853699, total loss: 1.832828\n",
      "0.36\n",
      "step 4500, entropy loss: 1.754907, l2_loss: 913.462769, total loss: 1.818850\n",
      "0.34\n",
      "step 4600, entropy loss: 1.626300, l2_loss: 914.058655, total loss: 1.690284\n",
      "0.35\n",
      "step 4700, entropy loss: 1.579230, l2_loss: 914.662659, total loss: 1.643256\n",
      "0.42\n",
      "step 4800, entropy loss: 1.544965, l2_loss: 915.236877, total loss: 1.609031\n",
      "0.37\n",
      "step 4900, entropy loss: 1.488840, l2_loss: 915.758118, total loss: 1.552943\n",
      "0.41\n",
      "step 5000, entropy loss: 1.601533, l2_loss: 916.331970, total loss: 1.665676\n",
      "0.37\n",
      "0.4421\n",
      "step 5100, entropy loss: 1.484879, l2_loss: 916.900085, total loss: 1.549062\n",
      "0.45\n",
      "step 5200, entropy loss: 1.441959, l2_loss: 917.471924, total loss: 1.506182\n",
      "0.47\n",
      "step 5300, entropy loss: 1.509165, l2_loss: 918.104736, total loss: 1.573433\n",
      "0.48\n",
      "step 5400, entropy loss: 1.528777, l2_loss: 918.749268, total loss: 1.593089\n",
      "0.51\n",
      "step 5500, entropy loss: 1.372118, l2_loss: 919.432983, total loss: 1.436478\n",
      "0.58\n",
      "step 5600, entropy loss: 1.387796, l2_loss: 920.203003, total loss: 1.452211\n",
      "0.55\n",
      "step 5700, entropy loss: 1.361077, l2_loss: 921.052368, total loss: 1.425551\n",
      "0.58\n",
      "step 5800, entropy loss: 1.161444, l2_loss: 921.938293, total loss: 1.225980\n",
      "0.64\n",
      "step 5900, entropy loss: 1.044867, l2_loss: 922.919983, total loss: 1.109472\n",
      "0.61\n",
      "step 6000, entropy loss: 1.148240, l2_loss: 923.898621, total loss: 1.212913\n",
      "0.6\n",
      "0.6502\n",
      "step 6100, entropy loss: 1.218827, l2_loss: 924.960815, total loss: 1.283574\n",
      "0.65\n",
      "step 6200, entropy loss: 0.952256, l2_loss: 926.006897, total loss: 1.017076\n",
      "0.79\n",
      "step 6300, entropy loss: 1.015030, l2_loss: 927.117188, total loss: 1.079928\n",
      "0.69\n",
      "step 6400, entropy loss: 0.939787, l2_loss: 928.208557, total loss: 1.004762\n",
      "0.73\n",
      "step 6500, entropy loss: 1.019035, l2_loss: 929.313416, total loss: 1.084087\n",
      "0.71\n",
      "step 6600, entropy loss: 0.857201, l2_loss: 930.336609, total loss: 0.922324\n",
      "0.66\n",
      "step 6700, entropy loss: 0.831043, l2_loss: 931.418091, total loss: 0.896242\n",
      "0.7\n",
      "step 6800, entropy loss: 0.713352, l2_loss: 932.416321, total loss: 0.778621\n",
      "0.8\n",
      "step 6900, entropy loss: 0.755012, l2_loss: 933.378662, total loss: 0.820349\n",
      "0.74\n",
      "step 7000, entropy loss: 0.647467, l2_loss: 934.298706, total loss: 0.712868\n",
      "0.82\n",
      "0.7898\n",
      "step 7100, entropy loss: 0.676931, l2_loss: 935.198669, total loss: 0.742395\n",
      "0.81\n",
      "step 7200, entropy loss: 0.876542, l2_loss: 936.046326, total loss: 0.942066\n",
      "0.75\n",
      "step 7300, entropy loss: 0.606314, l2_loss: 936.914124, total loss: 0.671898\n",
      "0.83\n",
      "step 7400, entropy loss: 0.655260, l2_loss: 937.801941, total loss: 0.720906\n",
      "0.78\n",
      "step 7500, entropy loss: 0.593683, l2_loss: 938.596069, total loss: 0.659384\n",
      "0.82\n",
      "step 7600, entropy loss: 0.528314, l2_loss: 939.351196, total loss: 0.594068\n",
      "0.9\n",
      "step 7700, entropy loss: 0.579536, l2_loss: 940.078491, total loss: 0.645341\n",
      "0.8\n",
      "step 7800, entropy loss: 0.675932, l2_loss: 940.755615, total loss: 0.741785\n",
      "0.82\n",
      "step 7900, entropy loss: 0.456920, l2_loss: 941.407837, total loss: 0.522819\n",
      "0.85\n",
      "step 8000, entropy loss: 0.494603, l2_loss: 942.112793, total loss: 0.560551\n",
      "0.85\n",
      "0.8399\n",
      "step 8100, entropy loss: 0.637136, l2_loss: 942.751221, total loss: 0.703129\n",
      "0.76\n",
      "step 8200, entropy loss: 0.431228, l2_loss: 943.415344, total loss: 0.497267\n",
      "0.88\n",
      "step 8300, entropy loss: 0.493882, l2_loss: 944.047852, total loss: 0.559965\n",
      "0.84\n",
      "step 8400, entropy loss: 0.633394, l2_loss: 944.605591, total loss: 0.699517\n",
      "0.85\n",
      "step 8500, entropy loss: 0.729504, l2_loss: 945.192993, total loss: 0.795668\n",
      "0.8\n",
      "step 8600, entropy loss: 0.540690, l2_loss: 945.811646, total loss: 0.606897\n",
      "0.91\n",
      "step 8700, entropy loss: 0.547231, l2_loss: 946.346313, total loss: 0.613476\n",
      "0.9\n",
      "step 8800, entropy loss: 0.394871, l2_loss: 946.862061, total loss: 0.461152\n",
      "0.89\n",
      "step 8900, entropy loss: 0.570270, l2_loss: 947.365540, total loss: 0.636586\n",
      "0.78\n",
      "step 9000, entropy loss: 0.472293, l2_loss: 947.871033, total loss: 0.538644\n",
      "0.89\n",
      "0.8721\n",
      "step 9100, entropy loss: 0.430870, l2_loss: 948.392456, total loss: 0.497258\n",
      "0.89\n",
      "step 9200, entropy loss: 0.470510, l2_loss: 948.898193, total loss: 0.536933\n",
      "0.83\n",
      "step 9300, entropy loss: 0.466128, l2_loss: 949.370911, total loss: 0.532584\n",
      "0.9\n",
      "step 9400, entropy loss: 0.319083, l2_loss: 949.824402, total loss: 0.385571\n",
      "0.92\n",
      "step 9500, entropy loss: 0.371694, l2_loss: 950.261292, total loss: 0.438213\n",
      "0.92\n",
      "step 9600, entropy loss: 0.430326, l2_loss: 950.709351, total loss: 0.496876\n",
      "0.88\n",
      "step 9700, entropy loss: 0.380242, l2_loss: 951.087646, total loss: 0.446818\n",
      "0.88\n",
      "step 9800, entropy loss: 0.332465, l2_loss: 951.526794, total loss: 0.399071\n",
      "0.93\n",
      "step 9900, entropy loss: 0.331798, l2_loss: 951.959595, total loss: 0.398435\n",
      "0.91\n",
      "step 10000, entropy loss: 0.291012, l2_loss: 952.354004, total loss: 0.357676\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.95\n",
      "0.8902\n",
      "step 10100, entropy loss: 0.316855, l2_loss: 952.783264, total loss: 0.383550\n",
      "0.91\n",
      "step 10200, entropy loss: 0.356695, l2_loss: 953.182739, total loss: 0.423417\n",
      "0.89\n",
      "step 10300, entropy loss: 0.401249, l2_loss: 953.576660, total loss: 0.467999\n",
      "0.92\n",
      "step 10400, entropy loss: 0.302413, l2_loss: 953.966614, total loss: 0.369191\n",
      "0.9\n",
      "step 10500, entropy loss: 0.394726, l2_loss: 954.320068, total loss: 0.461529\n",
      "0.86\n",
      "step 10600, entropy loss: 0.531655, l2_loss: 954.652649, total loss: 0.598481\n",
      "0.88\n",
      "step 10700, entropy loss: 0.277018, l2_loss: 955.015625, total loss: 0.343869\n",
      "0.94\n",
      "step 10800, entropy loss: 0.381899, l2_loss: 955.311829, total loss: 0.448771\n",
      "0.93\n",
      "step 10900, entropy loss: 0.375290, l2_loss: 955.664551, total loss: 0.442187\n",
      "0.9\n",
      "step 11000, entropy loss: 0.336108, l2_loss: 956.020630, total loss: 0.403030\n",
      "0.93\n",
      "0.8965\n",
      "step 11100, entropy loss: 0.360647, l2_loss: 956.328613, total loss: 0.427590\n",
      "0.9\n",
      "step 11200, entropy loss: 0.285150, l2_loss: 956.636719, total loss: 0.352115\n",
      "0.88\n",
      "step 11300, entropy loss: 0.372641, l2_loss: 956.941101, total loss: 0.439627\n",
      "0.91\n",
      "step 11400, entropy loss: 0.306488, l2_loss: 957.263672, total loss: 0.373497\n",
      "0.93\n",
      "step 11500, entropy loss: 0.506629, l2_loss: 957.556152, total loss: 0.573658\n",
      "0.89\n",
      "step 11600, entropy loss: 0.413384, l2_loss: 957.852051, total loss: 0.480434\n",
      "0.89\n",
      "step 11700, entropy loss: 0.290520, l2_loss: 958.095581, total loss: 0.357586\n",
      "0.95\n",
      "step 11800, entropy loss: 0.237523, l2_loss: 958.410522, total loss: 0.304612\n",
      "0.91\n",
      "step 11900, entropy loss: 0.297563, l2_loss: 958.671021, total loss: 0.364670\n",
      "0.93\n",
      "step 12000, entropy loss: 0.474614, l2_loss: 958.978638, total loss: 0.541742\n",
      "0.86\n",
      "0.9156\n",
      "step 12100, entropy loss: 0.361103, l2_loss: 959.241943, total loss: 0.428250\n",
      "0.92\n",
      "step 12200, entropy loss: 0.568411, l2_loss: 959.513062, total loss: 0.635577\n",
      "0.85\n",
      "step 12300, entropy loss: 0.325974, l2_loss: 959.746887, total loss: 0.393157\n",
      "0.93\n",
      "step 12400, entropy loss: 0.363259, l2_loss: 959.981201, total loss: 0.430458\n",
      "0.9\n",
      "step 12500, entropy loss: 0.268206, l2_loss: 960.248901, total loss: 0.335423\n",
      "0.94\n",
      "step 12600, entropy loss: 0.385754, l2_loss: 960.534180, total loss: 0.452992\n",
      "0.9\n",
      "step 12700, entropy loss: 0.265925, l2_loss: 960.759644, total loss: 0.333178\n",
      "0.92\n",
      "step 12800, entropy loss: 0.252530, l2_loss: 961.010986, total loss: 0.319801\n",
      "0.95\n",
      "step 12900, entropy loss: 0.166333, l2_loss: 961.243042, total loss: 0.233620\n",
      "0.96\n",
      "step 13000, entropy loss: 0.266037, l2_loss: 961.480469, total loss: 0.333341\n",
      "0.92\n",
      "0.9246\n",
      "step 13100, entropy loss: 0.241993, l2_loss: 961.706421, total loss: 0.309312\n",
      "0.91\n",
      "step 13200, entropy loss: 0.265694, l2_loss: 961.923462, total loss: 0.333028\n",
      "0.94\n",
      "step 13300, entropy loss: 0.176183, l2_loss: 962.132690, total loss: 0.243533\n",
      "0.97\n",
      "step 13400, entropy loss: 0.274970, l2_loss: 962.351318, total loss: 0.342335\n",
      "0.93\n",
      "step 13500, entropy loss: 0.268908, l2_loss: 962.541504, total loss: 0.336286\n",
      "0.94\n",
      "step 13600, entropy loss: 0.245543, l2_loss: 962.763062, total loss: 0.312937\n",
      "0.92\n",
      "step 13700, entropy loss: 0.309514, l2_loss: 962.943115, total loss: 0.376920\n",
      "0.94\n",
      "step 13800, entropy loss: 0.197201, l2_loss: 963.177673, total loss: 0.264624\n",
      "0.96\n",
      "step 13900, entropy loss: 0.175649, l2_loss: 963.387695, total loss: 0.243086\n",
      "0.96\n",
      "step 14000, entropy loss: 0.347104, l2_loss: 963.554749, total loss: 0.414553\n",
      "0.89\n",
      "0.9246\n",
      "step 14100, entropy loss: 0.154943, l2_loss: 963.748657, total loss: 0.222406\n",
      "0.98\n",
      "step 14200, entropy loss: 0.184976, l2_loss: 963.979370, total loss: 0.252455\n",
      "1.0\n",
      "step 14300, entropy loss: 0.335562, l2_loss: 964.149109, total loss: 0.403053\n",
      "0.9\n",
      "step 14400, entropy loss: 0.181092, l2_loss: 964.337036, total loss: 0.248596\n",
      "0.93\n",
      "step 14500, entropy loss: 0.416363, l2_loss: 964.528870, total loss: 0.483880\n",
      "0.92\n",
      "step 14600, entropy loss: 0.176315, l2_loss: 964.713135, total loss: 0.243845\n",
      "0.96\n",
      "step 14700, entropy loss: 0.292768, l2_loss: 964.883667, total loss: 0.360310\n",
      "0.92\n",
      "step 14800, entropy loss: 0.232631, l2_loss: 965.089600, total loss: 0.300187\n",
      "0.98\n",
      "step 14900, entropy loss: 0.402005, l2_loss: 965.245850, total loss: 0.469572\n",
      "0.92\n",
      "step 15000, entropy loss: 0.296435, l2_loss: 965.401978, total loss: 0.364013\n",
      "0.91\n",
      "0.9292\n",
      "step 15100, entropy loss: 0.174753, l2_loss: 965.573914, total loss: 0.242344\n",
      "0.98\n",
      "step 15200, entropy loss: 0.138560, l2_loss: 965.767090, total loss: 0.206163\n",
      "0.97\n",
      "step 15300, entropy loss: 0.233986, l2_loss: 965.925293, total loss: 0.301600\n",
      "0.9\n",
      "step 15400, entropy loss: 0.247896, l2_loss: 966.100830, total loss: 0.315523\n",
      "0.93\n",
      "step 15500, entropy loss: 0.096197, l2_loss: 966.232544, total loss: 0.163833\n",
      "0.95\n",
      "step 15600, entropy loss: 0.262974, l2_loss: 966.405457, total loss: 0.330622\n",
      "0.95\n",
      "step 15700, entropy loss: 0.338475, l2_loss: 966.557800, total loss: 0.406134\n",
      "0.9\n",
      "step 15800, entropy loss: 0.269289, l2_loss: 966.683838, total loss: 0.336957\n",
      "0.97\n",
      "step 15900, entropy loss: 0.209020, l2_loss: 966.873962, total loss: 0.276702\n",
      "0.95\n",
      "step 16000, entropy loss: 0.184400, l2_loss: 967.042114, total loss: 0.252093\n",
      "0.97\n",
      "0.9334\n",
      "step 16100, entropy loss: 0.409785, l2_loss: 967.161560, total loss: 0.477487\n",
      "0.93\n",
      "step 16200, entropy loss: 0.269186, l2_loss: 967.330933, total loss: 0.336899\n",
      "0.91\n",
      "step 16300, entropy loss: 0.108960, l2_loss: 967.488525, total loss: 0.176684\n",
      "0.99\n",
      "step 16400, entropy loss: 0.182921, l2_loss: 967.629700, total loss: 0.250655\n",
      "0.96\n",
      "step 16500, entropy loss: 0.251126, l2_loss: 967.775146, total loss: 0.318870\n",
      "0.95\n",
      "step 16600, entropy loss: 0.302492, l2_loss: 967.887634, total loss: 0.370244\n",
      "0.92\n",
      "step 16700, entropy loss: 0.258232, l2_loss: 968.026489, total loss: 0.325994\n",
      "0.89\n",
      "step 16800, entropy loss: 0.232587, l2_loss: 968.166992, total loss: 0.300359\n",
      "0.95\n",
      "step 16900, entropy loss: 0.336789, l2_loss: 968.305420, total loss: 0.404570\n",
      "0.92\n",
      "step 17000, entropy loss: 0.233554, l2_loss: 968.429199, total loss: 0.301344\n",
      "0.96\n",
      "0.9324\n",
      "step 17100, entropy loss: 0.147090, l2_loss: 968.565369, total loss: 0.214889\n",
      "0.95\n",
      "step 17200, entropy loss: 0.237797, l2_loss: 968.675415, total loss: 0.305604\n",
      "0.93\n",
      "step 17300, entropy loss: 0.191745, l2_loss: 968.824280, total loss: 0.259563\n",
      "0.92\n",
      "step 17400, entropy loss: 0.221151, l2_loss: 968.933594, total loss: 0.288976\n",
      "0.97\n",
      "step 17500, entropy loss: 0.196273, l2_loss: 969.065918, total loss: 0.264108\n",
      "0.96\n",
      "step 17600, entropy loss: 0.240739, l2_loss: 969.187439, total loss: 0.308583\n",
      "0.93\n",
      "step 17700, entropy loss: 0.162453, l2_loss: 969.297241, total loss: 0.230304\n",
      "0.98\n",
      "step 17800, entropy loss: 0.193458, l2_loss: 969.440430, total loss: 0.261319\n",
      "0.97\n",
      "step 17900, entropy loss: 0.165065, l2_loss: 969.569458, total loss: 0.232935\n",
      "0.96\n",
      "step 18000, entropy loss: 0.194257, l2_loss: 969.680054, total loss: 0.262135\n",
      "0.92\n",
      "0.9427\n",
      "step 18100, entropy loss: 0.233207, l2_loss: 969.810303, total loss: 0.301093\n",
      "0.96\n",
      "step 18200, entropy loss: 0.220587, l2_loss: 969.923584, total loss: 0.288482\n",
      "0.95\n",
      "step 18300, entropy loss: 0.187443, l2_loss: 970.014038, total loss: 0.255344\n",
      "0.97\n",
      "step 18400, entropy loss: 0.195538, l2_loss: 970.134399, total loss: 0.263447\n",
      "0.99\n",
      "step 18500, entropy loss: 0.239798, l2_loss: 970.257446, total loss: 0.307716\n",
      "0.94\n",
      "step 18600, entropy loss: 0.179469, l2_loss: 970.359497, total loss: 0.247394\n",
      "0.97\n",
      "step 18700, entropy loss: 0.227639, l2_loss: 970.445618, total loss: 0.295570\n",
      "0.96\n",
      "step 18800, entropy loss: 0.268967, l2_loss: 970.576172, total loss: 0.336908\n",
      "0.93\n",
      "step 18900, entropy loss: 0.171841, l2_loss: 970.668701, total loss: 0.239788\n",
      "0.96\n",
      "step 19000, entropy loss: 0.198049, l2_loss: 970.797668, total loss: 0.266005\n",
      "0.95\n",
      "0.9444\n",
      "step 19100, entropy loss: 0.151820, l2_loss: 970.895142, total loss: 0.219783\n",
      "0.95\n",
      "step 19200, entropy loss: 0.188867, l2_loss: 970.973511, total loss: 0.256836\n",
      "0.96\n",
      "step 19300, entropy loss: 0.178337, l2_loss: 971.077026, total loss: 0.246313\n",
      "0.97\n",
      "step 19400, entropy loss: 0.297445, l2_loss: 971.188782, total loss: 0.365428\n",
      "0.92\n",
      "step 19500, entropy loss: 0.264205, l2_loss: 971.292603, total loss: 0.332196\n",
      "0.96\n",
      "step 19600, entropy loss: 0.152492, l2_loss: 971.382690, total loss: 0.220489\n",
      "0.93\n",
      "step 19700, entropy loss: 0.313603, l2_loss: 971.495422, total loss: 0.381607\n",
      "0.96\n",
      "step 19800, entropy loss: 0.121390, l2_loss: 971.593506, total loss: 0.189402\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.96\n",
      "step 19900, entropy loss: 0.175691, l2_loss: 971.679749, total loss: 0.243709\n",
      "0.94\n",
      "step 20000, entropy loss: 0.290885, l2_loss: 971.771423, total loss: 0.358909\n",
      "0.93\n",
      "0.9462\n",
      "step 20100, entropy loss: 0.225914, l2_loss: 971.872681, total loss: 0.293945\n",
      "0.92\n",
      "step 20200, entropy loss: 0.203089, l2_loss: 971.981262, total loss: 0.271128\n",
      "0.95\n",
      "step 20300, entropy loss: 0.257300, l2_loss: 972.057678, total loss: 0.325344\n",
      "0.91\n",
      "step 20400, entropy loss: 0.179980, l2_loss: 972.140625, total loss: 0.248030\n",
      "0.97\n",
      "step 20500, entropy loss: 0.147331, l2_loss: 972.246216, total loss: 0.215388\n",
      "0.97\n",
      "step 20600, entropy loss: 0.168254, l2_loss: 972.332764, total loss: 0.236317\n",
      "0.96\n",
      "step 20700, entropy loss: 0.150501, l2_loss: 972.404419, total loss: 0.218569\n",
      "0.96\n",
      "step 20800, entropy loss: 0.166046, l2_loss: 972.509155, total loss: 0.234121\n",
      "0.95\n",
      "step 20900, entropy loss: 0.123597, l2_loss: 972.589111, total loss: 0.191678\n",
      "0.97\n",
      "step 21000, entropy loss: 0.138072, l2_loss: 972.667847, total loss: 0.206159\n",
      "0.98\n",
      "0.9475\n",
      "step 21100, entropy loss: 0.130792, l2_loss: 972.758057, total loss: 0.198885\n",
      "0.96\n",
      "step 21200, entropy loss: 0.190793, l2_loss: 972.839478, total loss: 0.258892\n",
      "0.98\n",
      "step 21300, entropy loss: 0.197995, l2_loss: 972.933716, total loss: 0.266100\n",
      "0.95\n",
      "step 21400, entropy loss: 0.078826, l2_loss: 973.028809, total loss: 0.146939\n",
      "0.99\n",
      "step 21500, entropy loss: 0.182056, l2_loss: 973.104553, total loss: 0.250173\n",
      "0.97\n",
      "step 21600, entropy loss: 0.291525, l2_loss: 973.200195, total loss: 0.359649\n",
      "0.97\n",
      "step 21700, entropy loss: 0.163995, l2_loss: 973.270691, total loss: 0.232124\n",
      "0.95\n",
      "step 21800, entropy loss: 0.162349, l2_loss: 973.353760, total loss: 0.230484\n",
      "0.98\n",
      "step 21900, entropy loss: 0.101229, l2_loss: 973.434814, total loss: 0.169370\n",
      "0.99\n",
      "step 22000, entropy loss: 0.235261, l2_loss: 973.542114, total loss: 0.303409\n",
      "0.96\n",
      "0.9462\n",
      "step 22100, entropy loss: 0.156978, l2_loss: 973.616821, total loss: 0.225132\n",
      "0.96\n",
      "step 22200, entropy loss: 0.205446, l2_loss: 973.672729, total loss: 0.273603\n",
      "0.94\n",
      "step 22300, entropy loss: 0.128270, l2_loss: 973.772278, total loss: 0.196434\n",
      "0.95\n",
      "step 22400, entropy loss: 0.153186, l2_loss: 973.867676, total loss: 0.221357\n",
      "0.96\n",
      "step 22500, entropy loss: 0.068785, l2_loss: 973.919434, total loss: 0.136959\n",
      "0.99\n",
      "step 22600, entropy loss: 0.309434, l2_loss: 973.982544, total loss: 0.377613\n",
      "0.94\n",
      "step 22700, entropy loss: 0.212415, l2_loss: 974.057922, total loss: 0.280599\n",
      "0.92\n",
      "step 22800, entropy loss: 0.114332, l2_loss: 974.140137, total loss: 0.182522\n",
      "0.96\n",
      "step 22900, entropy loss: 0.221576, l2_loss: 974.216553, total loss: 0.289772\n",
      "0.92\n",
      "step 23000, entropy loss: 0.181636, l2_loss: 974.305786, total loss: 0.249837\n",
      "0.94\n",
      "0.9507\n",
      "step 23100, entropy loss: 0.180920, l2_loss: 974.364929, total loss: 0.249126\n",
      "0.96\n",
      "step 23200, entropy loss: 0.144201, l2_loss: 974.419678, total loss: 0.212410\n",
      "0.97\n",
      "step 23300, entropy loss: 0.130222, l2_loss: 974.492004, total loss: 0.198436\n",
      "0.95\n",
      "step 23400, entropy loss: 0.156070, l2_loss: 974.579834, total loss: 0.224290\n",
      "0.96\n",
      "step 23500, entropy loss: 0.081373, l2_loss: 974.654175, total loss: 0.149599\n",
      "1.0\n",
      "step 23600, entropy loss: 0.075246, l2_loss: 974.704041, total loss: 0.143476\n",
      "1.0\n",
      "step 23700, entropy loss: 0.163195, l2_loss: 974.764465, total loss: 0.231429\n",
      "0.97\n",
      "step 23800, entropy loss: 0.220899, l2_loss: 974.830139, total loss: 0.289137\n",
      "0.97\n",
      "step 23900, entropy loss: 0.159350, l2_loss: 974.902710, total loss: 0.227593\n",
      "0.96\n",
      "step 24000, entropy loss: 0.194314, l2_loss: 974.974792, total loss: 0.262562\n",
      "0.95\n",
      "0.9459\n",
      "step 24100, entropy loss: 0.220199, l2_loss: 975.066711, total loss: 0.288454\n",
      "0.94\n",
      "step 24200, entropy loss: 0.052894, l2_loss: 975.135559, total loss: 0.121153\n",
      "0.98\n",
      "step 24300, entropy loss: 0.235469, l2_loss: 975.174500, total loss: 0.303731\n",
      "0.94\n",
      "step 24400, entropy loss: 0.089715, l2_loss: 975.260193, total loss: 0.157983\n",
      "0.98\n",
      "step 24500, entropy loss: 0.147349, l2_loss: 975.307129, total loss: 0.215620\n",
      "0.94\n",
      "step 24600, entropy loss: 0.112208, l2_loss: 975.367859, total loss: 0.180484\n",
      "0.97\n",
      "step 24700, entropy loss: 0.228031, l2_loss: 975.430481, total loss: 0.296311\n",
      "0.95\n",
      "step 24800, entropy loss: 0.259876, l2_loss: 975.502869, total loss: 0.328161\n",
      "0.94\n",
      "step 24900, entropy loss: 0.069356, l2_loss: 975.569336, total loss: 0.137646\n",
      "1.0\n",
      "step 25000, entropy loss: 0.250703, l2_loss: 975.598022, total loss: 0.318995\n",
      "0.94\n",
      "0.9515\n",
      "step 25100, entropy loss: 0.198374, l2_loss: 975.670898, total loss: 0.266671\n",
      "0.97\n",
      "step 25200, entropy loss: 0.191218, l2_loss: 975.747314, total loss: 0.259520\n",
      "0.92\n",
      "step 25300, entropy loss: 0.126859, l2_loss: 975.779663, total loss: 0.195163\n",
      "0.97\n",
      "step 25400, entropy loss: 0.250143, l2_loss: 975.844116, total loss: 0.318452\n",
      "0.96\n",
      "step 25500, entropy loss: 0.202423, l2_loss: 975.887146, total loss: 0.270735\n",
      "0.94\n",
      "step 25600, entropy loss: 0.301037, l2_loss: 975.950928, total loss: 0.369354\n",
      "0.92\n",
      "step 25700, entropy loss: 0.071072, l2_loss: 975.990417, total loss: 0.139391\n",
      "0.97\n",
      "step 25800, entropy loss: 0.121461, l2_loss: 976.054749, total loss: 0.189785\n",
      "0.94\n",
      "step 25900, entropy loss: 0.229293, l2_loss: 976.103760, total loss: 0.297620\n",
      "0.96\n",
      "step 26000, entropy loss: 0.217148, l2_loss: 976.151245, total loss: 0.285479\n",
      "0.96\n",
      "0.9516\n",
      "step 26100, entropy loss: 0.214205, l2_loss: 976.209229, total loss: 0.282540\n",
      "0.93\n",
      "step 26200, entropy loss: 0.093576, l2_loss: 976.262085, total loss: 0.161914\n",
      "0.99\n",
      "step 26300, entropy loss: 0.157250, l2_loss: 976.323486, total loss: 0.225593\n",
      "0.94\n",
      "step 26400, entropy loss: 0.167977, l2_loss: 976.370117, total loss: 0.236323\n",
      "0.98\n",
      "step 26500, entropy loss: 0.241442, l2_loss: 976.414490, total loss: 0.309791\n",
      "0.93\n",
      "step 26600, entropy loss: 0.128204, l2_loss: 976.485718, total loss: 0.196558\n",
      "0.96\n",
      "step 26700, entropy loss: 0.192785, l2_loss: 976.530823, total loss: 0.261142\n",
      "0.95\n",
      "step 26800, entropy loss: 0.077366, l2_loss: 976.572693, total loss: 0.145726\n",
      "0.98\n",
      "step 26900, entropy loss: 0.221172, l2_loss: 976.610779, total loss: 0.289535\n",
      "0.94\n",
      "step 27000, entropy loss: 0.274538, l2_loss: 976.643433, total loss: 0.342903\n",
      "0.93\n",
      "0.9543\n",
      "step 27100, entropy loss: 0.206900, l2_loss: 976.698608, total loss: 0.275268\n",
      "0.95\n",
      "step 27200, entropy loss: 0.114869, l2_loss: 976.747314, total loss: 0.183242\n",
      "0.98\n",
      "step 27300, entropy loss: 0.089720, l2_loss: 976.814209, total loss: 0.158097\n",
      "0.97\n",
      "step 27400, entropy loss: 0.208036, l2_loss: 976.860596, total loss: 0.276416\n",
      "0.94\n",
      "step 27500, entropy loss: 0.228069, l2_loss: 976.901733, total loss: 0.296452\n",
      "0.93\n",
      "step 27600, entropy loss: 0.055883, l2_loss: 976.968262, total loss: 0.124271\n",
      "0.97\n",
      "step 27700, entropy loss: 0.251349, l2_loss: 977.006775, total loss: 0.319740\n",
      "0.94\n",
      "step 27800, entropy loss: 0.120041, l2_loss: 977.057068, total loss: 0.188435\n",
      "0.97\n",
      "step 27900, entropy loss: 0.176800, l2_loss: 977.091064, total loss: 0.245197\n",
      "0.95\n",
      "step 28000, entropy loss: 0.097153, l2_loss: 977.147766, total loss: 0.165553\n",
      "0.98\n",
      "0.9556\n",
      "step 28100, entropy loss: 0.173939, l2_loss: 977.201721, total loss: 0.242343\n",
      "0.96\n",
      "step 28200, entropy loss: 0.073639, l2_loss: 977.233704, total loss: 0.142045\n",
      "0.99\n",
      "step 28300, entropy loss: 0.228711, l2_loss: 977.257874, total loss: 0.297119\n",
      "0.94\n",
      "step 28400, entropy loss: 0.117410, l2_loss: 977.317139, total loss: 0.185822\n",
      "0.97\n",
      "step 28500, entropy loss: 0.159259, l2_loss: 977.352783, total loss: 0.227674\n",
      "0.98\n",
      "step 28600, entropy loss: 0.117098, l2_loss: 977.384094, total loss: 0.185515\n",
      "0.95\n",
      "step 28700, entropy loss: 0.116392, l2_loss: 977.421143, total loss: 0.184811\n",
      "0.99\n",
      "step 28800, entropy loss: 0.211860, l2_loss: 977.479126, total loss: 0.280283\n",
      "0.95\n",
      "step 28900, entropy loss: 0.154665, l2_loss: 977.503845, total loss: 0.223090\n",
      "0.96\n",
      "step 29000, entropy loss: 0.088562, l2_loss: 977.533936, total loss: 0.156990\n",
      "0.98\n",
      "0.9608\n",
      "step 29100, entropy loss: 0.345860, l2_loss: 977.569214, total loss: 0.414290\n",
      "0.9\n",
      "step 29200, entropy loss: 0.246468, l2_loss: 977.614014, total loss: 0.314901\n",
      "0.93\n",
      "step 29300, entropy loss: 0.120310, l2_loss: 977.641357, total loss: 0.188745\n",
      "0.96\n",
      "step 29400, entropy loss: 0.180292, l2_loss: 977.709778, total loss: 0.248732\n",
      "0.93\n",
      "step 29500, entropy loss: 0.260198, l2_loss: 977.762573, total loss: 0.328642\n",
      "0.92\n",
      "step 29600, entropy loss: 0.108659, l2_loss: 977.784424, total loss: 0.177104\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.97\n",
      "step 29700, entropy loss: 0.097083, l2_loss: 977.826782, total loss: 0.165531\n",
      "0.98\n",
      "step 29800, entropy loss: 0.223051, l2_loss: 977.863037, total loss: 0.291501\n",
      "0.96\n",
      "step 29900, entropy loss: 0.282353, l2_loss: 977.869568, total loss: 0.350804\n",
      "0.95\n",
      "step 30000, entropy loss: 0.122665, l2_loss: 977.926270, total loss: 0.191120\n",
      "0.97\n",
      "0.96\n",
      "step 30100, entropy loss: 0.122785, l2_loss: 977.955811, total loss: 0.191242\n",
      "0.97\n",
      "step 30200, entropy loss: 0.212273, l2_loss: 978.014282, total loss: 0.280734\n",
      "0.95\n",
      "step 30300, entropy loss: 0.057802, l2_loss: 978.034180, total loss: 0.126265\n",
      "0.99\n",
      "step 30400, entropy loss: 0.169761, l2_loss: 978.078308, total loss: 0.238227\n",
      "0.96\n",
      "step 30500, entropy loss: 0.279571, l2_loss: 978.109497, total loss: 0.348038\n",
      "0.94\n",
      "step 30600, entropy loss: 0.149273, l2_loss: 978.146301, total loss: 0.217744\n",
      "0.96\n",
      "step 30700, entropy loss: 0.129395, l2_loss: 978.185669, total loss: 0.197868\n",
      "0.98\n",
      "step 30800, entropy loss: 0.108150, l2_loss: 978.222412, total loss: 0.176625\n",
      "0.97\n",
      "step 30900, entropy loss: 0.211364, l2_loss: 978.255676, total loss: 0.279842\n",
      "0.95\n",
      "step 31000, entropy loss: 0.118027, l2_loss: 978.272400, total loss: 0.186506\n",
      "0.98\n",
      "0.96\n",
      "step 31100, entropy loss: 0.169475, l2_loss: 978.311890, total loss: 0.237957\n",
      "0.95\n",
      "step 31200, entropy loss: 0.194777, l2_loss: 978.338013, total loss: 0.263261\n",
      "0.95\n",
      "step 31300, entropy loss: 0.076911, l2_loss: 978.375366, total loss: 0.145397\n",
      "0.99\n",
      "step 31400, entropy loss: 0.172688, l2_loss: 978.400391, total loss: 0.241176\n",
      "0.98\n",
      "step 31500, entropy loss: 0.093956, l2_loss: 978.423645, total loss: 0.162446\n",
      "0.98\n",
      "step 31600, entropy loss: 0.055517, l2_loss: 978.448486, total loss: 0.124009\n",
      "1.0\n",
      "step 31700, entropy loss: 0.098358, l2_loss: 978.511719, total loss: 0.166854\n",
      "0.98\n",
      "step 31800, entropy loss: 0.129048, l2_loss: 978.512512, total loss: 0.197544\n",
      "0.97\n",
      "step 31900, entropy loss: 0.246078, l2_loss: 978.549622, total loss: 0.314576\n",
      "0.95\n",
      "step 32000, entropy loss: 0.081074, l2_loss: 978.575317, total loss: 0.149575\n",
      "0.99\n",
      "0.9607\n",
      "step 32100, entropy loss: 0.232636, l2_loss: 978.608215, total loss: 0.301139\n",
      "0.94\n",
      "step 32200, entropy loss: 0.089631, l2_loss: 978.645996, total loss: 0.158136\n",
      "0.98\n",
      "step 32300, entropy loss: 0.079269, l2_loss: 978.671570, total loss: 0.147776\n",
      "0.99\n",
      "step 32400, entropy loss: 0.084821, l2_loss: 978.695557, total loss: 0.153330\n",
      "0.99\n",
      "step 32500, entropy loss: 0.173432, l2_loss: 978.729492, total loss: 0.241943\n",
      "0.94\n",
      "step 32600, entropy loss: 0.185242, l2_loss: 978.740723, total loss: 0.253753\n",
      "0.95\n",
      "step 32700, entropy loss: 0.156751, l2_loss: 978.761169, total loss: 0.225265\n",
      "0.96\n",
      "step 32800, entropy loss: 0.095482, l2_loss: 978.761536, total loss: 0.163995\n",
      "0.98\n",
      "step 32900, entropy loss: 0.105829, l2_loss: 978.800049, total loss: 0.174345\n",
      "0.96\n",
      "step 33000, entropy loss: 0.112265, l2_loss: 978.843140, total loss: 0.180784\n",
      "0.96\n",
      "0.9613\n",
      "step 33100, entropy loss: 0.089435, l2_loss: 978.879150, total loss: 0.157957\n",
      "1.0\n",
      "step 33200, entropy loss: 0.035547, l2_loss: 978.902649, total loss: 0.104070\n",
      "0.99\n",
      "step 33300, entropy loss: 0.108517, l2_loss: 978.931213, total loss: 0.177042\n",
      "0.97\n",
      "step 33400, entropy loss: 0.165198, l2_loss: 978.958740, total loss: 0.233726\n",
      "0.96\n",
      "step 33500, entropy loss: 0.074180, l2_loss: 978.983765, total loss: 0.142709\n",
      "1.0\n",
      "step 33600, entropy loss: 0.127784, l2_loss: 979.000183, total loss: 0.196314\n",
      "0.94\n",
      "step 33700, entropy loss: 0.104799, l2_loss: 979.037720, total loss: 0.173332\n",
      "0.96\n",
      "step 33800, entropy loss: 0.113117, l2_loss: 979.059509, total loss: 0.181651\n",
      "0.96\n",
      "step 33900, entropy loss: 0.213235, l2_loss: 979.067383, total loss: 0.281770\n",
      "0.94\n",
      "step 34000, entropy loss: 0.101428, l2_loss: 979.102539, total loss: 0.169965\n",
      "0.98\n",
      "0.9611\n",
      "step 34100, entropy loss: 0.224865, l2_loss: 979.133911, total loss: 0.293404\n",
      "0.94\n",
      "step 34200, entropy loss: 0.076925, l2_loss: 979.181213, total loss: 0.145467\n",
      "0.97\n",
      "step 34300, entropy loss: 0.120524, l2_loss: 979.193237, total loss: 0.189068\n",
      "0.97\n",
      "step 34400, entropy loss: 0.141520, l2_loss: 979.205261, total loss: 0.210065\n",
      "0.96\n",
      "step 34500, entropy loss: 0.177251, l2_loss: 979.258667, total loss: 0.245799\n",
      "0.97\n",
      "step 34600, entropy loss: 0.098229, l2_loss: 979.280701, total loss: 0.166778\n",
      "0.96\n",
      "step 34700, entropy loss: 0.186901, l2_loss: 979.307983, total loss: 0.255452\n",
      "0.93\n",
      "step 34800, entropy loss: 0.107133, l2_loss: 979.314087, total loss: 0.175685\n",
      "0.99\n",
      "step 34900, entropy loss: 0.199706, l2_loss: 979.316833, total loss: 0.268259\n",
      "0.95\n",
      "step 35000, entropy loss: 0.083895, l2_loss: 979.329224, total loss: 0.152448\n",
      "0.98\n",
      "0.9631\n",
      "step 35100, entropy loss: 0.091994, l2_loss: 979.339905, total loss: 0.160548\n",
      "0.96\n",
      "step 35200, entropy loss: 0.126879, l2_loss: 979.384338, total loss: 0.195436\n",
      "0.94\n",
      "step 35300, entropy loss: 0.089420, l2_loss: 979.404419, total loss: 0.157978\n",
      "0.97\n",
      "step 35400, entropy loss: 0.138535, l2_loss: 979.432251, total loss: 0.207095\n",
      "0.94\n",
      "step 35500, entropy loss: 0.108130, l2_loss: 979.465515, total loss: 0.176692\n",
      "0.98\n",
      "step 35600, entropy loss: 0.141134, l2_loss: 979.507935, total loss: 0.209700\n",
      "0.97\n",
      "step 35700, entropy loss: 0.155668, l2_loss: 979.485840, total loss: 0.224232\n",
      "0.94\n",
      "step 35800, entropy loss: 0.100451, l2_loss: 979.500610, total loss: 0.169016\n",
      "0.97\n",
      "step 35900, entropy loss: 0.120874, l2_loss: 979.514404, total loss: 0.189440\n",
      "0.98\n",
      "step 36000, entropy loss: 0.124615, l2_loss: 979.534180, total loss: 0.193182\n",
      "0.96\n",
      "0.9629\n",
      "step 36100, entropy loss: 0.130813, l2_loss: 979.555176, total loss: 0.199382\n",
      "0.96\n",
      "step 36200, entropy loss: 0.139499, l2_loss: 979.570923, total loss: 0.208069\n",
      "0.98\n",
      "step 36300, entropy loss: 0.217045, l2_loss: 979.595764, total loss: 0.285616\n",
      "0.97\n",
      "step 36400, entropy loss: 0.202731, l2_loss: 979.596375, total loss: 0.271303\n",
      "0.93\n",
      "step 36500, entropy loss: 0.119113, l2_loss: 979.621460, total loss: 0.187687\n",
      "0.98\n",
      "step 36600, entropy loss: 0.165120, l2_loss: 979.641357, total loss: 0.233695\n",
      "0.98\n",
      "step 36700, entropy loss: 0.077307, l2_loss: 979.678589, total loss: 0.145885\n",
      "0.99\n",
      "step 36800, entropy loss: 0.088006, l2_loss: 979.681030, total loss: 0.156583\n",
      "0.99\n",
      "step 36900, entropy loss: 0.102382, l2_loss: 979.706482, total loss: 0.170961\n",
      "0.98\n",
      "step 37000, entropy loss: 0.122280, l2_loss: 979.702393, total loss: 0.190859\n",
      "0.96\n",
      "0.9675\n",
      "step 37100, entropy loss: 0.085303, l2_loss: 979.712708, total loss: 0.153883\n",
      "0.98\n",
      "step 37200, entropy loss: 0.210252, l2_loss: 979.735718, total loss: 0.278833\n",
      "0.96\n",
      "step 37300, entropy loss: 0.203019, l2_loss: 979.760986, total loss: 0.271602\n",
      "0.97\n",
      "step 37400, entropy loss: 0.093937, l2_loss: 979.764099, total loss: 0.162521\n",
      "0.98\n",
      "step 37500, entropy loss: 0.150768, l2_loss: 979.770569, total loss: 0.219352\n",
      "0.98\n",
      "step 37600, entropy loss: 0.055864, l2_loss: 979.785522, total loss: 0.124449\n",
      "0.99\n",
      "step 37700, entropy loss: 0.045675, l2_loss: 979.815369, total loss: 0.114262\n",
      "1.0\n",
      "step 37800, entropy loss: 0.266528, l2_loss: 979.821167, total loss: 0.335116\n",
      "0.95\n",
      "step 37900, entropy loss: 0.128582, l2_loss: 979.842896, total loss: 0.197171\n",
      "0.97\n",
      "step 38000, entropy loss: 0.301791, l2_loss: 979.854126, total loss: 0.370381\n",
      "0.94\n",
      "0.9685\n",
      "step 38100, entropy loss: 0.054543, l2_loss: 979.854736, total loss: 0.123132\n",
      "1.0\n",
      "step 38200, entropy loss: 0.123475, l2_loss: 979.881714, total loss: 0.192067\n",
      "0.96\n",
      "step 38300, entropy loss: 0.100718, l2_loss: 979.892578, total loss: 0.169311\n",
      "0.97\n",
      "step 38400, entropy loss: 0.124257, l2_loss: 979.910156, total loss: 0.192851\n",
      "0.98\n",
      "step 38500, entropy loss: 0.215973, l2_loss: 979.907166, total loss: 0.284566\n",
      "0.95\n",
      "step 38600, entropy loss: 0.110566, l2_loss: 979.932861, total loss: 0.179162\n",
      "0.97\n",
      "step 38700, entropy loss: 0.267969, l2_loss: 979.960571, total loss: 0.336566\n",
      "0.93\n",
      "step 38800, entropy loss: 0.070048, l2_loss: 979.964111, total loss: 0.138646\n",
      "0.99\n",
      "step 38900, entropy loss: 0.123899, l2_loss: 979.998596, total loss: 0.192499\n",
      "0.98\n",
      "step 39000, entropy loss: 0.139794, l2_loss: 979.984375, total loss: 0.208393\n",
      "0.96\n",
      "0.9672\n",
      "step 39100, entropy loss: 0.111038, l2_loss: 979.973145, total loss: 0.179636\n",
      "0.97\n",
      "step 39200, entropy loss: 0.041059, l2_loss: 980.007324, total loss: 0.109660\n",
      "1.0\n",
      "step 39300, entropy loss: 0.102751, l2_loss: 980.058533, total loss: 0.171355\n",
      "0.95\n",
      "step 39400, entropy loss: 0.072992, l2_loss: 980.062439, total loss: 0.141596\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "1.0\n",
      "step 39500, entropy loss: 0.157414, l2_loss: 980.077026, total loss: 0.226019\n",
      "0.96\n",
      "step 39600, entropy loss: 0.114603, l2_loss: 980.092163, total loss: 0.183210\n",
      "0.97\n",
      "step 39700, entropy loss: 0.103241, l2_loss: 980.094666, total loss: 0.171847\n",
      "0.97\n",
      "step 39800, entropy loss: 0.124308, l2_loss: 980.104858, total loss: 0.192916\n",
      "0.96\n",
      "step 39900, entropy loss: 0.130430, l2_loss: 980.121521, total loss: 0.199038\n",
      "0.95\n",
      "step 40000, entropy loss: 0.164489, l2_loss: 980.118530, total loss: 0.233098\n",
      "0.95\n",
      "0.9661\n",
      "step 40100, entropy loss: 0.102701, l2_loss: 980.140198, total loss: 0.171311\n",
      "0.97\n",
      "step 40200, entropy loss: 0.086336, l2_loss: 980.146912, total loss: 0.154947\n",
      "0.98\n",
      "step 40300, entropy loss: 0.140668, l2_loss: 980.153503, total loss: 0.209278\n",
      "0.95\n",
      "step 40400, entropy loss: 0.161630, l2_loss: 980.174438, total loss: 0.230243\n",
      "0.94\n",
      "step 40500, entropy loss: 0.072008, l2_loss: 980.186890, total loss: 0.140621\n",
      "0.98\n",
      "step 40600, entropy loss: 0.102326, l2_loss: 980.197937, total loss: 0.170940\n",
      "0.98\n",
      "step 40700, entropy loss: 0.261675, l2_loss: 980.212585, total loss: 0.330290\n",
      "0.93\n",
      "step 40800, entropy loss: 0.129710, l2_loss: 980.196411, total loss: 0.198324\n",
      "0.97\n",
      "step 40900, entropy loss: 0.089906, l2_loss: 980.221924, total loss: 0.158522\n",
      "0.99\n",
      "step 41000, entropy loss: 0.108296, l2_loss: 980.210388, total loss: 0.176910\n",
      "0.96\n",
      "0.9669\n",
      "step 41100, entropy loss: 0.147408, l2_loss: 980.203125, total loss: 0.216022\n",
      "0.98\n",
      "step 41200, entropy loss: 0.072433, l2_loss: 980.231445, total loss: 0.141050\n",
      "0.98\n",
      "step 41300, entropy loss: 0.219274, l2_loss: 980.226074, total loss: 0.287890\n",
      "0.95\n",
      "step 41400, entropy loss: 0.140557, l2_loss: 980.241699, total loss: 0.209174\n",
      "0.96\n",
      "step 41500, entropy loss: 0.129959, l2_loss: 980.249268, total loss: 0.198577\n",
      "0.97\n",
      "step 41600, entropy loss: 0.087804, l2_loss: 980.265198, total loss: 0.156422\n",
      "0.98\n",
      "step 41700, entropy loss: 0.076656, l2_loss: 980.287354, total loss: 0.145276\n",
      "0.97\n",
      "step 41800, entropy loss: 0.084760, l2_loss: 980.298828, total loss: 0.153381\n",
      "0.96\n",
      "step 41900, entropy loss: 0.194429, l2_loss: 980.305481, total loss: 0.263051\n",
      "0.94\n",
      "step 42000, entropy loss: 0.137474, l2_loss: 980.280823, total loss: 0.206093\n",
      "0.97\n",
      "0.9646\n",
      "step 42100, entropy loss: 0.127174, l2_loss: 980.283203, total loss: 0.195794\n",
      "0.97\n",
      "step 42200, entropy loss: 0.094962, l2_loss: 980.301880, total loss: 0.163583\n",
      "1.0\n",
      "step 42300, entropy loss: 0.081527, l2_loss: 980.297241, total loss: 0.150148\n",
      "1.0\n",
      "step 42400, entropy loss: 0.120451, l2_loss: 980.312683, total loss: 0.189073\n",
      "0.98\n",
      "step 42500, entropy loss: 0.075443, l2_loss: 980.311401, total loss: 0.144065\n",
      "0.99\n",
      "step 42600, entropy loss: 0.047635, l2_loss: 980.308105, total loss: 0.116256\n",
      "0.99\n",
      "step 42700, entropy loss: 0.108281, l2_loss: 980.319214, total loss: 0.176904\n",
      "0.99\n",
      "step 42800, entropy loss: 0.120224, l2_loss: 980.315857, total loss: 0.188846\n",
      "0.96\n",
      "step 42900, entropy loss: 0.066526, l2_loss: 980.330383, total loss: 0.135149\n",
      "0.98\n",
      "step 43000, entropy loss: 0.073780, l2_loss: 980.342529, total loss: 0.142404\n",
      "0.99\n",
      "0.9696\n",
      "step 43100, entropy loss: 0.112302, l2_loss: 980.346924, total loss: 0.180926\n",
      "0.97\n",
      "step 43200, entropy loss: 0.075504, l2_loss: 980.352295, total loss: 0.144128\n",
      "1.0\n",
      "step 43300, entropy loss: 0.096551, l2_loss: 980.366333, total loss: 0.165176\n",
      "0.98\n",
      "step 43400, entropy loss: 0.086474, l2_loss: 980.368835, total loss: 0.155100\n",
      "0.99\n",
      "step 43500, entropy loss: 0.110865, l2_loss: 980.375427, total loss: 0.179491\n",
      "0.99\n",
      "step 43600, entropy loss: 0.130140, l2_loss: 980.373108, total loss: 0.198766\n",
      "0.98\n",
      "step 43700, entropy loss: 0.109179, l2_loss: 980.388977, total loss: 0.177807\n",
      "0.95\n",
      "step 43800, entropy loss: 0.085355, l2_loss: 980.407043, total loss: 0.153983\n",
      "0.96\n",
      "step 43900, entropy loss: 0.077901, l2_loss: 980.403503, total loss: 0.146529\n",
      "0.98\n",
      "step 44000, entropy loss: 0.188538, l2_loss: 980.393982, total loss: 0.257165\n",
      "0.98\n",
      "0.9686\n",
      "step 44100, entropy loss: 0.063988, l2_loss: 980.384521, total loss: 0.132615\n",
      "0.99\n",
      "step 44200, entropy loss: 0.125989, l2_loss: 980.394531, total loss: 0.194617\n",
      "0.95\n",
      "step 44300, entropy loss: 0.191180, l2_loss: 980.416626, total loss: 0.259809\n",
      "0.97\n",
      "step 44400, entropy loss: 0.153518, l2_loss: 980.398010, total loss: 0.222146\n",
      "0.97\n",
      "step 44500, entropy loss: 0.099000, l2_loss: 980.389709, total loss: 0.167627\n",
      "0.96\n",
      "step 44600, entropy loss: 0.128482, l2_loss: 980.389160, total loss: 0.197109\n",
      "0.97\n",
      "step 44700, entropy loss: 0.087351, l2_loss: 980.407104, total loss: 0.155979\n",
      "0.98\n",
      "step 44800, entropy loss: 0.108529, l2_loss: 980.428223, total loss: 0.177159\n",
      "0.98\n",
      "step 44900, entropy loss: 0.079189, l2_loss: 980.401611, total loss: 0.147817\n",
      "0.99\n",
      "step 45000, entropy loss: 0.128491, l2_loss: 980.401794, total loss: 0.197119\n",
      "0.97\n",
      "0.9703\n",
      "step 45100, entropy loss: 0.075034, l2_loss: 980.412598, total loss: 0.143663\n",
      "0.99\n",
      "step 45200, entropy loss: 0.034306, l2_loss: 980.403442, total loss: 0.102934\n",
      "1.0\n",
      "step 45300, entropy loss: 0.155078, l2_loss: 980.403137, total loss: 0.223706\n",
      "0.97\n",
      "step 45400, entropy loss: 0.111789, l2_loss: 980.393127, total loss: 0.180417\n",
      "0.98\n",
      "step 45500, entropy loss: 0.075748, l2_loss: 980.412781, total loss: 0.144377\n",
      "0.99\n",
      "step 45600, entropy loss: 0.121687, l2_loss: 980.419556, total loss: 0.190316\n",
      "0.96\n",
      "step 45700, entropy loss: 0.082655, l2_loss: 980.414978, total loss: 0.151284\n",
      "0.98\n",
      "step 45800, entropy loss: 0.032766, l2_loss: 980.408386, total loss: 0.101394\n",
      "1.0\n",
      "step 45900, entropy loss: 0.019902, l2_loss: 980.409241, total loss: 0.088531\n",
      "1.0\n",
      "step 46000, entropy loss: 0.176552, l2_loss: 980.415710, total loss: 0.245181\n",
      "0.98\n",
      "0.9706\n",
      "step 46100, entropy loss: 0.054775, l2_loss: 980.426575, total loss: 0.123405\n",
      "1.0\n",
      "step 46200, entropy loss: 0.058086, l2_loss: 980.424011, total loss: 0.126716\n",
      "0.98\n",
      "step 46300, entropy loss: 0.071443, l2_loss: 980.404419, total loss: 0.140071\n",
      "0.98\n",
      "step 46400, entropy loss: 0.073905, l2_loss: 980.415527, total loss: 0.142534\n",
      "0.96\n",
      "step 46500, entropy loss: 0.156878, l2_loss: 980.419678, total loss: 0.225508\n",
      "0.95\n",
      "step 46600, entropy loss: 0.072248, l2_loss: 980.421387, total loss: 0.140878\n",
      "0.99\n",
      "step 46700, entropy loss: 0.187102, l2_loss: 980.411011, total loss: 0.255730\n",
      "0.95\n",
      "step 46800, entropy loss: 0.034357, l2_loss: 980.422363, total loss: 0.102987\n",
      "1.0\n",
      "step 46900, entropy loss: 0.194525, l2_loss: 980.417297, total loss: 0.263155\n",
      "0.95\n",
      "step 47000, entropy loss: 0.086239, l2_loss: 980.408081, total loss: 0.154868\n",
      "0.99\n",
      "0.9727\n",
      "step 47100, entropy loss: 0.094032, l2_loss: 980.399658, total loss: 0.162660\n",
      "0.97\n",
      "step 47200, entropy loss: 0.100409, l2_loss: 980.407776, total loss: 0.169038\n",
      "0.99\n",
      "step 47300, entropy loss: 0.084936, l2_loss: 980.405884, total loss: 0.153564\n",
      "0.98\n",
      "step 47400, entropy loss: 0.040405, l2_loss: 980.397522, total loss: 0.109033\n",
      "0.99\n",
      "step 47500, entropy loss: 0.067823, l2_loss: 980.369263, total loss: 0.136449\n",
      "0.98\n",
      "step 47600, entropy loss: 0.077820, l2_loss: 980.383118, total loss: 0.146447\n",
      "0.99\n",
      "step 47700, entropy loss: 0.074911, l2_loss: 980.376038, total loss: 0.143538\n",
      "0.98\n",
      "step 47800, entropy loss: 0.056315, l2_loss: 980.369202, total loss: 0.124941\n",
      "0.99\n",
      "step 47900, entropy loss: 0.039282, l2_loss: 980.362305, total loss: 0.107907\n",
      "1.0\n",
      "step 48000, entropy loss: 0.096696, l2_loss: 980.362488, total loss: 0.165321\n",
      "0.98\n",
      "0.9696\n",
      "step 48100, entropy loss: 0.082807, l2_loss: 980.349365, total loss: 0.151431\n",
      "0.98\n",
      "step 48200, entropy loss: 0.168970, l2_loss: 980.344116, total loss: 0.237595\n",
      "0.97\n",
      "step 48300, entropy loss: 0.020667, l2_loss: 980.317749, total loss: 0.089289\n",
      "1.0\n",
      "step 48400, entropy loss: 0.123766, l2_loss: 980.320435, total loss: 0.192388\n",
      "0.98\n",
      "step 48500, entropy loss: 0.157125, l2_loss: 980.326172, total loss: 0.225748\n",
      "0.96\n",
      "step 48600, entropy loss: 0.118483, l2_loss: 980.319580, total loss: 0.187106\n",
      "0.98\n",
      "step 48700, entropy loss: 0.277177, l2_loss: 980.329895, total loss: 0.345800\n",
      "0.96\n",
      "step 48800, entropy loss: 0.075666, l2_loss: 980.317871, total loss: 0.144289\n",
      "0.98\n",
      "step 48900, entropy loss: 0.038554, l2_loss: 980.302917, total loss: 0.107175\n",
      "0.99\n",
      "step 49000, entropy loss: 0.124157, l2_loss: 980.292542, total loss: 0.192777\n",
      "0.95\n",
      "0.973\n",
      "step 49100, entropy loss: 0.073052, l2_loss: 980.300293, total loss: 0.141673\n",
      "0.97\n",
      "step 49200, entropy loss: 0.045950, l2_loss: 980.268860, total loss: 0.114569\n",
      "1.0\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "step 49300, entropy loss: 0.076150, l2_loss: 980.271606, total loss: 0.144769\n",
      "0.98\n",
      "step 49400, entropy loss: 0.034750, l2_loss: 980.256104, total loss: 0.103368\n",
      "0.99\n",
      "step 49500, entropy loss: 0.084067, l2_loss: 980.268433, total loss: 0.152686\n",
      "0.99\n",
      "step 49600, entropy loss: 0.081426, l2_loss: 980.261963, total loss: 0.150045\n",
      "0.99\n",
      "step 49700, entropy loss: 0.075105, l2_loss: 980.263184, total loss: 0.143723\n",
      "0.98\n",
      "step 49800, entropy loss: 0.136648, l2_loss: 980.269653, total loss: 0.205267\n",
      "0.97\n",
      "step 49900, entropy loss: 0.207629, l2_loss: 980.258240, total loss: 0.276247\n",
      "0.92\n",
      "step 50000, entropy loss: 0.110803, l2_loss: 980.240662, total loss: 0.179419\n",
      "0.98\n",
      "0.9712\n",
      "step 50100, entropy loss: 0.211013, l2_loss: 980.229004, total loss: 0.279629\n",
      "0.97\n",
      "step 50200, entropy loss: 0.072113, l2_loss: 980.234375, total loss: 0.140730\n",
      "0.98\n",
      "step 50300, entropy loss: 0.033103, l2_loss: 980.235779, total loss: 0.101720\n",
      "1.0\n",
      "step 50400, entropy loss: 0.068903, l2_loss: 980.247437, total loss: 0.137520\n",
      "0.98\n",
      "step 50500, entropy loss: 0.105288, l2_loss: 980.245361, total loss: 0.173905\n",
      "0.97\n",
      "step 50600, entropy loss: 0.101932, l2_loss: 980.241028, total loss: 0.170549\n",
      "0.98\n",
      "step 50700, entropy loss: 0.145753, l2_loss: 980.237061, total loss: 0.214370\n",
      "0.97\n",
      "step 50800, entropy loss: 0.152672, l2_loss: 980.219727, total loss: 0.221287\n",
      "0.96\n",
      "step 50900, entropy loss: 0.133554, l2_loss: 980.201965, total loss: 0.202168\n",
      "0.94\n",
      "step 51000, entropy loss: 0.065074, l2_loss: 980.196533, total loss: 0.133687\n",
      "1.0\n",
      "0.9737\n",
      "step 51100, entropy loss: 0.086943, l2_loss: 980.194214, total loss: 0.155557\n",
      "0.98\n",
      "step 51200, entropy loss: 0.163008, l2_loss: 980.192444, total loss: 0.231621\n",
      "0.97\n",
      "step 51300, entropy loss: 0.104695, l2_loss: 980.179321, total loss: 0.173308\n",
      "0.98\n",
      "step 51400, entropy loss: 0.093707, l2_loss: 980.177856, total loss: 0.162320\n",
      "0.96\n",
      "step 51500, entropy loss: 0.056228, l2_loss: 980.167908, total loss: 0.124840\n",
      "0.98\n",
      "step 51600, entropy loss: 0.068652, l2_loss: 980.172485, total loss: 0.137264\n",
      "0.98\n",
      "step 51700, entropy loss: 0.087262, l2_loss: 980.154480, total loss: 0.155873\n",
      "0.98\n",
      "step 51800, entropy loss: 0.151151, l2_loss: 980.169678, total loss: 0.219763\n",
      "0.96\n",
      "step 51900, entropy loss: 0.086162, l2_loss: 980.148071, total loss: 0.154772\n",
      "0.98\n",
      "step 52000, entropy loss: 0.060010, l2_loss: 980.113647, total loss: 0.128618\n",
      "0.99\n",
      "0.9725\n",
      "step 52100, entropy loss: 0.131223, l2_loss: 980.119751, total loss: 0.199831\n",
      "0.97\n",
      "step 52200, entropy loss: 0.103272, l2_loss: 980.114380, total loss: 0.171880\n",
      "0.97\n",
      "step 52300, entropy loss: 0.102822, l2_loss: 980.121582, total loss: 0.171430\n",
      "0.97\n",
      "step 52400, entropy loss: 0.057369, l2_loss: 980.096497, total loss: 0.125975\n",
      "1.0\n",
      "step 52500, entropy loss: 0.113440, l2_loss: 980.085327, total loss: 0.182046\n",
      "0.96\n",
      "step 52600, entropy loss: 0.068401, l2_loss: 980.076721, total loss: 0.137006\n",
      "0.99\n",
      "step 52700, entropy loss: 0.049823, l2_loss: 980.054138, total loss: 0.118427\n",
      "0.99\n",
      "step 52800, entropy loss: 0.096525, l2_loss: 980.041992, total loss: 0.165128\n",
      "0.96\n",
      "step 52900, entropy loss: 0.097929, l2_loss: 980.038269, total loss: 0.166532\n",
      "0.97\n",
      "step 53000, entropy loss: 0.051031, l2_loss: 980.027710, total loss: 0.119633\n",
      "0.98\n",
      "0.9734\n",
      "step 53100, entropy loss: 0.076954, l2_loss: 980.012573, total loss: 0.145554\n",
      "0.97\n",
      "step 53200, entropy loss: 0.045464, l2_loss: 979.990051, total loss: 0.114063\n",
      "1.0\n",
      "step 53300, entropy loss: 0.075917, l2_loss: 979.996948, total loss: 0.144516\n",
      "0.98\n",
      "step 53400, entropy loss: 0.079108, l2_loss: 979.988831, total loss: 0.147707\n",
      "0.97\n",
      "step 53500, entropy loss: 0.169678, l2_loss: 979.963013, total loss: 0.238275\n",
      "0.98\n",
      "step 53600, entropy loss: 0.123910, l2_loss: 979.951782, total loss: 0.192506\n",
      "0.96\n",
      "step 53700, entropy loss: 0.197382, l2_loss: 979.939575, total loss: 0.265978\n",
      "0.95\n",
      "step 53800, entropy loss: 0.037773, l2_loss: 979.939331, total loss: 0.106369\n",
      "0.99\n",
      "step 53900, entropy loss: 0.054737, l2_loss: 979.922913, total loss: 0.123331\n",
      "0.98\n",
      "step 54000, entropy loss: 0.123915, l2_loss: 979.912354, total loss: 0.192509\n",
      "0.96\n",
      "0.9703\n",
      "step 54100, entropy loss: 0.121842, l2_loss: 979.902161, total loss: 0.190435\n",
      "0.97\n",
      "step 54200, entropy loss: 0.076767, l2_loss: 979.899902, total loss: 0.145360\n",
      "0.98\n",
      "step 54300, entropy loss: 0.190868, l2_loss: 979.890137, total loss: 0.259461\n",
      "0.97\n",
      "step 54400, entropy loss: 0.099576, l2_loss: 979.880432, total loss: 0.168167\n",
      "0.98\n",
      "step 54500, entropy loss: 0.056580, l2_loss: 979.858276, total loss: 0.125170\n",
      "0.99\n",
      "step 54600, entropy loss: 0.131831, l2_loss: 979.841064, total loss: 0.200420\n",
      "0.96\n",
      "step 54700, entropy loss: 0.116369, l2_loss: 979.816956, total loss: 0.184957\n",
      "0.96\n",
      "step 54800, entropy loss: 0.112779, l2_loss: 979.807007, total loss: 0.181365\n",
      "0.99\n",
      "step 54900, entropy loss: 0.016119, l2_loss: 979.808899, total loss: 0.084706\n",
      "1.0\n",
      "step 55000, entropy loss: 0.156319, l2_loss: 979.789307, total loss: 0.224904\n",
      "0.93\n",
      "0.9731\n",
      "step 55100, entropy loss: 0.059036, l2_loss: 979.769348, total loss: 0.127620\n",
      "0.98\n",
      "step 55200, entropy loss: 0.122923, l2_loss: 979.751099, total loss: 0.191506\n",
      "0.98\n",
      "step 55300, entropy loss: 0.056093, l2_loss: 979.748291, total loss: 0.124676\n",
      "1.0\n",
      "step 55400, entropy loss: 0.139189, l2_loss: 979.736145, total loss: 0.207770\n",
      "0.97\n",
      "step 55500, entropy loss: 0.068823, l2_loss: 979.719238, total loss: 0.137403\n",
      "1.0\n",
      "step 55600, entropy loss: 0.086550, l2_loss: 979.716797, total loss: 0.155130\n",
      "0.96\n",
      "step 55700, entropy loss: 0.254377, l2_loss: 979.698914, total loss: 0.322956\n",
      "0.97\n",
      "step 55800, entropy loss: 0.022658, l2_loss: 979.685181, total loss: 0.091236\n",
      "1.0\n",
      "step 55900, entropy loss: 0.020707, l2_loss: 979.675659, total loss: 0.089284\n",
      "1.0\n",
      "step 56000, entropy loss: 0.051663, l2_loss: 979.646729, total loss: 0.120238\n",
      "1.0\n",
      "0.9744\n",
      "step 56100, entropy loss: 0.039356, l2_loss: 979.638062, total loss: 0.107931\n",
      "1.0\n",
      "step 56200, entropy loss: 0.095490, l2_loss: 979.623901, total loss: 0.164063\n",
      "0.96\n",
      "step 56300, entropy loss: 0.058715, l2_loss: 979.628235, total loss: 0.127289\n",
      "0.97\n",
      "step 56400, entropy loss: 0.103507, l2_loss: 979.618408, total loss: 0.172080\n",
      "0.96\n",
      "step 56500, entropy loss: 0.121006, l2_loss: 979.603394, total loss: 0.189578\n",
      "0.97\n",
      "step 56600, entropy loss: 0.223511, l2_loss: 979.586060, total loss: 0.292082\n",
      "0.96\n",
      "step 56700, entropy loss: 0.121170, l2_loss: 979.576599, total loss: 0.189740\n",
      "0.97\n",
      "step 56800, entropy loss: 0.046655, l2_loss: 979.554810, total loss: 0.115224\n",
      "0.99\n",
      "step 56900, entropy loss: 0.049373, l2_loss: 979.544006, total loss: 0.117942\n",
      "0.99\n",
      "step 57000, entropy loss: 0.076915, l2_loss: 979.549927, total loss: 0.145484\n",
      "0.98\n",
      "0.9749\n",
      "step 57100, entropy loss: 0.109571, l2_loss: 979.530945, total loss: 0.178138\n",
      "0.97\n",
      "step 57200, entropy loss: 0.127603, l2_loss: 979.525146, total loss: 0.196170\n",
      "0.98\n",
      "step 57300, entropy loss: 0.067130, l2_loss: 979.497559, total loss: 0.135695\n",
      "0.97\n",
      "step 57400, entropy loss: 0.070715, l2_loss: 979.472656, total loss: 0.139278\n",
      "1.0\n",
      "step 57500, entropy loss: 0.146823, l2_loss: 979.467529, total loss: 0.215386\n",
      "0.95\n",
      "step 57600, entropy loss: 0.028878, l2_loss: 979.464722, total loss: 0.097441\n",
      "1.0\n",
      "step 57700, entropy loss: 0.087380, l2_loss: 979.430298, total loss: 0.155940\n",
      "0.98\n",
      "step 57800, entropy loss: 0.075261, l2_loss: 979.403320, total loss: 0.143819\n",
      "0.96\n",
      "step 57900, entropy loss: 0.090616, l2_loss: 979.380005, total loss: 0.159173\n",
      "0.98\n",
      "step 58000, entropy loss: 0.164011, l2_loss: 979.386047, total loss: 0.232568\n",
      "0.97\n",
      "0.9731\n",
      "step 58100, entropy loss: 0.150100, l2_loss: 979.344116, total loss: 0.218654\n",
      "0.97\n",
      "step 58200, entropy loss: 0.103113, l2_loss: 979.332703, total loss: 0.171666\n",
      "0.97\n",
      "step 58300, entropy loss: 0.097572, l2_loss: 979.324585, total loss: 0.166125\n",
      "0.98\n",
      "step 58400, entropy loss: 0.083814, l2_loss: 979.314331, total loss: 0.152366\n",
      "0.97\n",
      "step 58500, entropy loss: 0.023413, l2_loss: 979.304565, total loss: 0.091964\n",
      "1.0\n",
      "step 58600, entropy loss: 0.059511, l2_loss: 979.284485, total loss: 0.128061\n",
      "0.99\n",
      "step 58700, entropy loss: 0.124532, l2_loss: 979.259766, total loss: 0.193080\n",
      "0.96\n",
      "step 58800, entropy loss: 0.053931, l2_loss: 979.228760, total loss: 0.122477\n",
      "1.0\n",
      "step 58900, entropy loss: 0.052413, l2_loss: 979.230591, total loss: 0.120960\n",
      "1.0\n",
      "step 59000, entropy loss: 0.172357, l2_loss: 979.211426, total loss: 0.240902\n",
      "0.96\n",
      "0.9764\n",
      "step 59100, entropy loss: 0.060614, l2_loss: 979.198975, total loss: 0.129158\n"
     ]
    },
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.99\n",
      "step 59200, entropy loss: 0.039274, l2_loss: 979.189026, total loss: 0.107817\n",
      "0.99\n",
      "step 59300, entropy loss: 0.107279, l2_loss: 979.186157, total loss: 0.175822\n",
      "0.97\n",
      "step 59400, entropy loss: 0.244152, l2_loss: 979.190063, total loss: 0.312695\n",
      "0.92\n",
      "step 59500, entropy loss: 0.117858, l2_loss: 979.154419, total loss: 0.186399\n",
      "0.97\n",
      "step 59600, entropy loss: 0.048503, l2_loss: 979.135132, total loss: 0.117043\n",
      "0.99\n",
      "step 59700, entropy loss: 0.044455, l2_loss: 979.116028, total loss: 0.112993\n",
      "1.0\n",
      "step 59800, entropy loss: 0.081535, l2_loss: 979.098572, total loss: 0.150072\n",
      "0.98\n",
      "step 59900, entropy loss: 0.115094, l2_loss: 979.111084, total loss: 0.183631\n",
      "0.99\n",
      "step 60000, entropy loss: 0.094509, l2_loss: 979.084351, total loss: 0.163045\n",
      "0.99\n",
      "0.9748\n"
     ]
    }
   ],
   "source": [
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "with tf.name_scope('reshape'):\n",
    "  x_image = tf.reshape(x, [-1, 28, 28, 1])\n",
    "\n",
    "# First convolutional layer - maps one grayscale image to 32 feature maps.\n",
    "with tf.name_scope('conv1'):\n",
    "  h_conv1 = tf.contrib.slim.conv2d(x_image, 32, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "\n",
    "# Pooling layer - downsamples by 2X.\n",
    "with tf.name_scope('pool1'):\n",
    "  h_pool1 = tf.contrib.slim.max_pool2d(h_conv1, [2,2], stride=2, \n",
    "                         padding='VALID')\n",
    "\n",
    "# Second convolutional layer -- maps 32 feature maps to 64.\n",
    "with tf.name_scope('conv2'):\n",
    "  h_conv2 = tf.contrib.slim.conv2d(h_pool1, 64, [5,5],\n",
    "                             padding='SAME',\n",
    "                             activation_fn=tf.nn.relu)\n",
    "\n",
    "# Second pooling layer.\n",
    "with tf.name_scope('pool2'):\n",
    "  h_pool2 = tf.contrib.slim.max_pool2d(h_conv2, [2,2],\n",
    "                        stride=[2, 2], padding='VALID')\n",
    "\n",
    "\n",
    "# Fully connected layer 1 -- after 2 round of downsampling, our 28x28 image\n",
    "# is down to 7x7x64 feature maps -- maps this to 1024 features.\n",
    "with tf.name_scope('fc1'):\n",
    "  h_pool2_flat = tf.contrib.slim.avg_pool2d(h_pool2, h_pool2.shape[1:3],\n",
    "                        stride=[1, 1], padding='VALID')\n",
    "  h_fc1 = tf.contrib.slim.conv2d(h_pool2_flat, 1024, [1,1], activation_fn=tf.nn.relu)\n",
    "\n",
    "# Dropout - controls the complexity of the model, prevents co-adaptation of\n",
    "# features.\n",
    "with tf.name_scope('dropout'):\n",
    "  keep_prob = tf.placeholder(tf.float32)\n",
    "  h_fc1_drop = tf.nn.dropout(h_fc1, keep_prob)\n",
    "\n",
    "# Map the 1024 features to 10 classes, one for each digit\n",
    "with tf.name_scope('fc2'):\n",
    "  y = tf.squeeze(tf.contrib.slim.conv2d(h_fc1_drop, 10, [1,1], activation_fn=None))\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "\n",
    "l2_loss = tf.add_n( [tf.nn.l2_loss(w) for w in tf.get_collection(tf.GraphKeys.TRAINABLE_VARIABLES)] )\n",
    "total_loss = cross_entropy + 7e-5*l2_loss\n",
    "train_step = tf.train.GradientDescentOptimizer(learning_rate).minimize(total_loss)\n",
    "\n",
    "sess = tf.Session()\n",
    "init_op = tf.global_variables_initializer()\n",
    "sess.run(init_op)\n",
    "# Train\n",
    "for step in range(60000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 0.01\n",
    "  _, loss, l2_loss_value, total_loss_value = sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, learning_rate:lr, keep_prob:0.5})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    # Test trained model\n",
    "    correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "    accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "    print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys, keep_prob:0.5}))\n",
    "  if (step+1) % 1000 == 0:\n",
    "    print(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels, keep_prob:0.5}))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.5"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
